Package Gnumed :: Package pycommon :: Module gmPG2 |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmPG2

       1  """GNUmed PostgreSQL connection handling.
       2
       3  TODO: iterator/generator batch fetching:
       4          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
       5          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
       6
       7  winner:
       8  def resultset_functional_batchgenerator(cursor, size=100):
       9          for results in iter(lambda: cursor.fetchmany(size), []):
      10                  for rec in results:
      11                          yield rec
      12  """
      13  # =======================================================================
      14  __author__  = "K.Hilbert <Karsten.Hilbert@gmx.net>"
      15  __license__ = 'GPL v2 or later (details at http://www.gnu.org)'
      16
      17  # stdlib
      18  import time
      19  import sys
      20  import os
      21  import stat
      22  import io
      23  import codecs
      24  import logging
      25  import datetime as pydt
      26  import re as regex
      27  import threading
      28  import hashlib
      29  import shutil
      30
      31
      32  # GNUmed
      33  if __name__ == '__main__':
      34          sys.path.insert(0, '../../')
      35  from Gnumed.pycommon import gmLoginInfo
      36  from Gnumed.pycommon import gmExceptions
      37  from Gnumed.pycommon import gmDateTime
      38  from Gnumed.pycommon import gmBorg
      39  from Gnumed.pycommon import gmI18N
      40  from Gnumed.pycommon import gmLog2
      41  from Gnumed.pycommon import gmTools
      42  from Gnumed.pycommon.gmTools import prompted_input, u_replacement_character, format_dict_like
      43
      44  _log = logging.getLogger('gm.db')
      45
      46
      47  # 3rd party
      48  try:
      49          import psycopg2 as dbapi
      50  except ImportError:
      51          _log.exception("Python database adapter psycopg2 not found.")
      52          print("CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server.")
      53          raise
      54
      55
      56  _log.info('psycopg2 version: %s' % dbapi.__version__)
      57  _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
      58  if not (float(dbapi.apilevel) >= 2.0):
      59          raise ImportError('gmPG2: supported DB-API level too low')
      60  if not (dbapi.threadsafety > 0):
      61          raise ImportError('gmPG2: lacking minimum thread safety in psycopg2')
      62  if not (dbapi.paramstyle == 'pyformat'):
      63          raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2')
      64  try:
      65          dbapi.__version__.index('dt')
      66  except ValueError:
      67          raise ImportError('gmPG2: lacking datetime support in psycopg2')
      68  try:
      69          dbapi.__version__.index('ext')
      70  except ValueError:
      71          raise ImportError('gmPG2: lacking extensions support in psycopg2')
      72  try:
      73          dbapi.__version__.index('pq3')
      74  except ValueError:
      75          raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2')
      76
      77  import psycopg2.extras
      78  import psycopg2.extensions
      79  import psycopg2.pool
      80  import psycopg2.errorcodes as sql_error_codes
      81
      82  PG_ERROR_EXCEPTION = dbapi.Error
      83
      84  # =======================================================================
      85  _default_client_encoding = 'UTF8'
      86  _log.info('assuming default client encoding of [%s]' % _default_client_encoding)
      87
      88  # things timezone
      89  _default_client_timezone = None                 # default time zone for connections
      90  _sql_set_timezone = None
      91  _timestamp_template = "cast('%s' as timestamp with time zone)"          # MUST NOT be uniocde or else getquoted will not work
      92  FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone
      93
      94  _default_dsn = None
      95  _default_login = None
      96
      97  default_database = 'gnumed_v22'
      98
      99  postgresql_version_string = None
     100  postgresql_version = None                       # accuracy: major.minor
     101
     102  __ro_conn_pool = None
     103
     104  auto_request_login_params = True
     105
     106  # =======================================================================
     107  # global data
     108  # =======================================================================
     109
     110  known_schema_hashes = {
     111          0: 'not released, testing only',
     112          2: 'b09d50d7ed3f91ddf4c4ddb8ea507720',
     113          3: 'e73718eaf230d8f1d2d01afa8462e176',
     114          4: '4428ccf2e54c289136819e701bb095ea',
     115          5: '7e7b093af57aea48c288e76632a382e5',  # ... old (v1) style hashes
     116          6: '90e2026ac2efd236da9c8608b8685b2d',  # new (v2) style hashes ...
     117          7: '6c9f6d3981483f8e9433df99d1947b27',
     118          8: '89b13a7af83337c3aad153b717e52360',
     119          9: '641a9b2be3c378ffc2bb2f0b1c9f051d',
     120          10: '7ef42a8fb2bd929a2cdd0c63864b4e8a',
     121          11: '03042ae24f3f92877d986fb0a6184d76',
     122          12: '06183a6616db62257e22814007a8ed07',
     123          13: 'fab7c1ae408a6530c47f9b5111a0841e',
     124          14: 'e170d543f067d1ea60bfe9076b1560cf',
     125          15: '70012ff960b77ecdff4981c94b5b55b6',
     126          16: '0bcf44ca22c479b52976e5eda1de8161',
     127          17: '161428ee97a00e3bf56168c3a15b7b50',
     128          18: 'a0f9efcabdecfb4ddb6d8c0b69c02092',
     129          #19: '419e5225259c53dd36ad80d82066ff02' # 19.0 only
     130          #19: '9765373098b03fb208332498f34cd4b5' # until 19.11
     131          19: '57f009a159f55f77525cc0291e0c8b60', # starting with 19.12
     132          20: 'baed1901ed4c2f272b56c8cb2c6d88e8',
     133          21: 'e6a51a89dd22b75b61ead8f7083f251f',
     134          22: 'bf45f01327fb5feb2f5d3c06ba4a6792'
     135  }
     136
     137  map_schema_hash2version = {
     138          'b09d50d7ed3f91ddf4c4ddb8ea507720': 2,
     139          'e73718eaf230d8f1d2d01afa8462e176': 3,
     140          '4428ccf2e54c289136819e701bb095ea': 4,
     141          '7e7b093af57aea48c288e76632a382e5': 5,
     142          '90e2026ac2efd236da9c8608b8685b2d': 6,
     143          '6c9f6d3981483f8e9433df99d1947b27': 7,
     144          '89b13a7af83337c3aad153b717e52360': 8,
     145          '641a9b2be3c378ffc2bb2f0b1c9f051d': 9,
     146          '7ef42a8fb2bd929a2cdd0c63864b4e8a': 10,
     147          '03042ae24f3f92877d986fb0a6184d76': 11,
     148          '06183a6616db62257e22814007a8ed07': 12,
     149          'fab7c1ae408a6530c47f9b5111a0841e': 13,
     150          'e170d543f067d1ea60bfe9076b1560cf': 14,
     151          '70012ff960b77ecdff4981c94b5b55b6': 15,
     152          '0bcf44ca22c479b52976e5eda1de8161': 16,
     153          '161428ee97a00e3bf56168c3a15b7b50': 17,
     154          'a0f9efcabdecfb4ddb6d8c0b69c02092': 18,
     155          #'419e5225259c53dd36ad80d82066ff02': 19 # 19.0 only
     156          #'9765373098b03fb208332498f34cd4b5': 19 # until 19.11
     157          '57f009a159f55f77525cc0291e0c8b60': 19, # starting with 19.12
     158          'baed1901ed4c2f272b56c8cb2c6d88e8': 20,
     159          'e6a51a89dd22b75b61ead8f7083f251f': 21,
     160          'bf45f01327fb5feb2f5d3c06ba4a6792': 22
     161  }
     162
     163  map_client_branch2required_db_version = {
     164          'GIT tree': 0,
     165          '0.3': 9,
     166          '0.4': 10,
     167          '0.5': 11,
     168          '0.6': 12,
     169          '0.7': 13,
     170          '0.8': 14,
     171          '0.9': 15,
     172          '1.0': 16,              # intentional duplicate with 1.1
     173          '1.1': 16,
     174          '1.2': 17,
     175          '1.3': 18,
     176          '1.4': 19,
     177          '1.5': 20,
     178          '1.6': 21,
     179          '1.7': 22,
     180          '1.8': 22               # Yes, SAME as 1.7, no DB change.
     181  }
     182
     183  map_psyco_tx_status2str = [
     184          'TRANSACTION_STATUS_IDLE',
     185          'TRANSACTION_STATUS_ACTIVE',
     186          'TRANSACTION_STATUS_INTRANS',
     187          'TRANSACTION_STATUS_INERROR',
     188          'TRANSACTION_STATUS_UNKNOWN'
     189  ]
     190
     191  map_psyco_conn_status2str = [
     192          '0 - ?',
     193          'STATUS_READY',
     194          'STATUS_BEGIN_ALIAS_IN_TRANSACTION',
     195          'STATUS_PREPARED'
     196  ]
     197
     198  map_psyco_iso_level2str = {
     199          None: 'ISOLATION_LEVEL_DEFAULT (configured on server)',
     200          0: 'ISOLATION_LEVEL_AUTOCOMMIT',
     201          1: 'ISOLATION_LEVEL_READ_UNCOMMITTED',
     202          2: 'ISOLATION_LEVEL_REPEATABLE_READ',
     203          3: 'ISOLATION_LEVEL_SERIALIZABLE',
     204          4: 'ISOLATION_LEVEL_READ_UNCOMMITTED'
     205  }
     206
     207  # get columns and data types for a given table
     208  query_table_col_defs = """select
     209          cols.column_name,
     210          cols.udt_name
     211  from
     212          information_schema.columns cols
     213  where
     214          cols.table_schema = %s
     215                  and
     216          cols.table_name = %s
     217  order by
     218          cols.ordinal_position"""
     219
     220  query_table_attributes = """select
     221          cols.column_name
     222  from
     223          information_schema.columns cols
     224  where
     225          cols.table_schema = %s
     226                  and
     227          cols.table_name = %s
     228  order by
     229          cols.ordinal_position"""
     230
     231  # only works for single-column FKs but that's fine
     232  # needs gm-dbo, any-doc won't work
     233  SQL_foreign_key_name = """SELECT
     234          fk_tbl.*,
     235          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = fk_tbl.connamespace) AS constraint_schema,
     236          fk_tbl.conname AS constraint_name,
     237          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.conrelid)) AS source_schema,
     238          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.conrelid) AS source_table,
     239          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.conkey[1] AND attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass) AS source_column,
     240          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.confrelid)) AS target_schema,
     241          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.confrelid) AS target_table,
     242          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.confkey[1] AND attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass) AS target_column
     243  FROM
     244          pg_catalog.pg_constraint fk_tbl
     245  WHERE
     246          fk_tbl.contype = 'f'
     247                  AND
     248          fk_tbl.conrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
     249                  AND
     250          fk_tbl.conkey[1] = (
     251                  SELECT
     252                          col_tbl1.attnum
     253                  FROM
     254                          pg_catalog.pg_attribute col_tbl1
     255                  WHERE
     256                          col_tbl1.attname = %(src_col)s
     257                                  AND
     258                          col_tbl1.attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
     259          )
     260                  AND
     261          fk_tbl.confrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
     262                  AND
     263          fk_tbl.confkey[1] = (
     264                  SELECT
     265                          col_tbl2.attnum
     266                  FROM
     267                          pg_catalog.pg_attribute col_tbl2
     268                  WHERE
     269                          col_tbl2.attname = %(target_col)s
     270                                  AND
     271                          col_tbl2.attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
     272          )
     273  """
     274
     275  SQL_get_index_name = """
     276  SELECT
     277          (SELECT nspname FROM pg_namespace WHERE pg_namespace.oid = pg_class.relnamespace)
     278                  AS index_schema,
     279          pg_class.relname
     280                  AS index_name
     281  FROM
     282          pg_class
     283  WHERE
     284          pg_class.oid IN (
     285                  SELECT
     286                          indexrelid
     287                  FROM
     288                          pg_index
     289                  WHERE
     290                          pg_index.indrelid = %(idx_tbl)s::regclass
     291                                  AND
     292                          pg_index.indnatts = 1           -- only one column in index
     293                                  AND
     294                          pg_index.indkey[0] IN (
     295                                  SELECT
     296                                          pg_attribute.attnum
     297                                  FROM
     298                                          pg_attribute
     299                                  WHERE
     300                                          pg_attribute.attrelid = %(idx_tbl)s::regclass
     301                                                  AND
     302                                          pg_attribute.attname = %(idx_col)s
     303                                  )
     304          )
     305  """
     306
     307  SQL_get_pk_col_def = """
     308  SELECT
     309          pg_attribute.attname
     310                  AS pk_col,
     311          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
     312                  AS pk_type
     313  FROM pg_index, pg_class, pg_attribute, pg_namespace
     314  WHERE
     315          pg_class.oid = %(table)s::regclass
     316                  AND
     317          indrelid = pg_class.oid
     318                  AND
     319  --      nspname = %%(schema)s
     320  --              AND
     321          pg_class.relnamespace = pg_namespace.oid
     322                  AND
     323          pg_attribute.attrelid = pg_class.oid
     324                  AND
     325          pg_attribute.attnum = any(pg_index.indkey)
     326                  AND
     327          indisprimary
     328  """
     329
     330  # =======================================================================
     331  # module globals API
     332  # =======================================================================
     333 -def set_default_client_encoding(encoding = None):
     334          # check whether psycopg2 can handle this encoding
     335          if encoding not in psycopg2.extensions.encodings:
     336                  raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding)
     337          # check whether Python can handle this encoding
     338          py_enc = psycopg2.extensions.encodings[encoding]
     339          try:
     340                  codecs.lookup(py_enc)
     341          except LookupError:
     342                  _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc))
     343                  raise
     344          # FIXME: check encoding against the database
     345          # FIXME: - but we may not yet have access
     346          # FIXME: - psycopg2 will pull its encodings from the database eventually
     347          # it seems save to set it
     348          global _default_client_encoding
     349          _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, encoding))
     350          _default_client_encoding = encoding
     351          return True
     352
     353  #---------------------------------------------------
     354 -def set_default_client_timezone(timezone = None):
     355
     356          # FIXME: use __validate
     357          global _default_client_timezone
     358          _log.info('setting default client time zone from [%s] to [%s]' % (_default_client_timezone, timezone))
     359          _default_client_timezone = timezone
     360
     361          global _sql_set_timezone
     362          _sql_set_timezone = 'set timezone to %s'
     363
     364          return True
     365
     366  #---------------------------------------------------
     367 -def __validate_timezone(conn=None, timezone=None):
     368
     369          _log.debug('validating time zone [%s]', timezone)
     370
     371          cmd = 'set timezone to %(tz)s'
     372          args = {'tz': timezone}
     373
     374          conn.commit()
     375          curs = conn.cursor()
     376          is_valid = False
     377          try:
     378                  curs.execute(cmd, args)
     379                  _log.info('time zone [%s] is settable', timezone)
     380                  # can we actually use it, though ?
     381                  cmd = """select '1920-01-19 23:00:00+01'::timestamp with time zone"""
     382                  try:
     383                          curs.execute(cmd)
     384                          curs.fetchone()
     385                          _log.info('time zone [%s] is usable', timezone)
     386                          is_valid = True
     387                  except:
     388                          _log.error('error using time zone [%s]', timezone)
     389          except dbapi.DataError:
     390                  _log.warning('time zone [%s] is not settable', timezone)
     391          except:
     392                  _log.error('failed to set time zone to [%s]', timezone)
     393                  _log.exception('')
     394
     395          curs.close()
     396          conn.rollback()
     397
     398          return is_valid
     399
     400  #---------------------------------------------------
     401 -def __expand_timezone(conn=None, timezone=None):
     402          """some timezone defs are abbreviations so try to expand
     403          them because "set time zone" doesn't take abbreviations"""
     404
     405          cmd = """
     406  select distinct on (abbrev) name
     407  from pg_timezone_names
     408  where
     409          abbrev = %(tz)s and
     410          name ~ '^[^/]+/[^/]+$' and
     411          name !~ '^Etc/'
     412  """
     413          args = {'tz': timezone}
     414
     415          conn.commit()
     416          curs = conn.cursor()
     417
     418          result = timezone
     419          try:
     420                  curs.execute(cmd, args)
     421                  rows = curs.fetchall()
     422                  if len(rows) > 0:
     423                          result = rows[0]['name']
     424                          _log.debug('[%s] maps to [%s]', timezone, result)
     425          except:
     426                  _log.exception('cannot expand timezone abbreviation [%s]', timezone)
     427
     428          curs.close()
     429          conn.rollback()
     430
     431          return result
     432
     433  #---------------------------------------------------
     434 -def __detect_client_timezone(conn=None):
     435          """This is run on the very first connection."""
     436
     437          # FIXME: check whether server.timezone is the same
     438          # FIXME: value as what we eventually detect
     439
     440          # we need gmDateTime to be initialized
     441          if gmDateTime.current_local_iso_numeric_timezone_string is None:
     442                  gmDateTime.init()
     443
     444          _log.debug('trying to detect timezone from system')
     445
     446          tz_candidates = []
     447          try:
     448                  tz = os.environ['TZ']
     449                  tz_candidates.append(tz)
     450                  expanded = __expand_timezone(conn = conn, timezone = tz)
     451                  if expanded != tz:
     452                          tz_candidates.append(expanded)
     453          except KeyError:
     454                  pass
     455
     456          tz_candidates.append(gmDateTime.current_local_timezone_name)
     457          expanded = __expand_timezone(conn = conn, timezone = gmDateTime.current_local_timezone_name)
     458          if expanded != gmDateTime.current_local_timezone_name:
     459                  tz_candidates.append(expanded)
     460
     461          _log.debug('candidates: %s', tz_candidates)
     462
     463          # find best among candidates
     464          global _default_client_timezone
     465          global _sql_set_timezone
     466          found = False
     467          for tz in tz_candidates:
     468                  if __validate_timezone(conn = conn, timezone = tz):
     469                          _default_client_timezone = tz
     470                          _sql_set_timezone = 'set timezone to %s'
     471                          found = True
     472                          break
     473
     474          if not found:
     475                  _default_client_timezone = gmDateTime.current_local_iso_numeric_timezone_string
     476                  _sql_set_timezone = "set time zone interval %s hour to minute"
     477
     478          _log.info('client system time zone detected as equivalent to [%s]', _default_client_timezone)
     479
     480  # =======================================================================
     481  # login API
     482  # =======================================================================
     483 -def __request_login_params_tui():
     484          """Text mode request of database login parameters"""
     485          import getpass
     486          login = gmLoginInfo.LoginInfo()
     487
     488          print("\nPlease enter the required login parameters:")
     489          try:
     490                  login.host = prompted_input(prompt = "host ('' = non-TCP/IP)", default = '')
     491                  login.database = prompted_input(prompt = "database", default = default_database)
     492                  login.user = prompted_input(prompt = "user name", default = '')
     493                  tmp = 'password for "%s" (not shown): ' % login.user
     494                  login.password = getpass.getpass(tmp)
     495                  gmLog2.add_word2hide(login.password)
     496                  login.port = prompted_input(prompt = "port", default = 5432)
     497          except KeyboardInterrupt:
     498                  _log.warning("user cancelled text mode login dialog")
     499                  print("user cancelled text mode login dialog")
     500                  raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!"))
     501
     502          return login
     503
     504  #---------------------------------------------------
     505 -def __request_login_params_gui_wx():
     506          """GUI (wx) input request for database login parameters.
     507
     508          Returns gmLoginInfo.LoginInfo object
     509          """
     510          import wx
     511          # OK, wxPython was already loaded. But has the main Application instance
     512          # been initialized yet ? if not, the exception will kick us out
     513          if wx.GetApp() is None:
     514                  raise AssertionError(_("The wxPython GUI framework hasn't been initialized yet!"))
     515
     516          # Let's launch the login dialog
     517          # if wx was not initialized /no main App loop, an exception should be raised anyway
     518          import gmAuthWidgets
     519          dlg = gmAuthWidgets.cLoginDialog(None, -1)
     520          dlg.ShowModal()
     521          login = dlg.panel.GetLoginInfo()
     522          dlg.DestroyLater()
     523
     524          #if user cancelled or something else went wrong, raise an exception
     525          if login is None:
     526                  raise gmExceptions.ConnectionError(_("Can't connect to database without login information!"))
     527
     528          gmLog2.add_word2hide(login.password)
     529
     530          return login
     531
     532  #---------------------------------------------------
     533 -def request_login_params():
     534          """Request login parameters for database connection."""
     535          # do we auto-request parameters at all ?
     536          if not auto_request_login_params:
     537                  raise Exception('Cannot request login parameters.')
     538
     539          # are we inside X ?
     540          # if we aren't wxGTK will crash hard at the C-level with "can't open Display"
     541          if 'DISPLAY' in os.environ:
     542                  # try wxPython GUI
     543                  try:
     544                          return __request_login_params_gui_wx()
     545                  except:
     546                          pass
     547
     548          # well, either we are on the console or
     549          # wxPython does not work, use text mode
     550          return __request_login_params_tui()
     551
     552  # =======================================================================
     553  # DSN API
     554  # -----------------------------------------------------------------------
     555 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
     556          dsn_parts = []
     557
     558          if (database is not None) and (database.strip() != ''):
     559                  dsn_parts.append('dbname=%s' % database)
     560
     561          if (host is not None) and (host.strip() != ''):
     562                  dsn_parts.append('host=%s' % host)
     563
     564          if (port is not None) and (str(port).strip() != ''):
     565                  dsn_parts.append('port=%s' % port)
     566
     567          if (user is not None) and (user.strip() != ''):
     568                  dsn_parts.append('user=%s' % user)
     569
     570          if (password is not None) and (password.strip() != ''):
     571                  dsn_parts.append('password=%s' % password)
     572
     573          dsn_parts.append('sslmode=prefer')
     574          dsn_parts.append('fallback_application_name=GNUmed')
     575
     576          return ' '.join(dsn_parts)
     577
     578  # ------------------------------------------------------
     579 -def get_default_login():
     580          # make sure we do have a login
     581          get_default_dsn()
     582          return _default_login
     583
     584  # ------------------------------------------------------
     585 -def get_default_dsn():
     586          global _default_dsn
     587          if _default_dsn is not None:
     588                  return _default_dsn
     589
     590          login = request_login_params()
     591          set_default_login(login=login)
     592
     593          return _default_dsn
     594
     595  # ------------------------------------------------------
     596 -def set_default_login(login=None):
     597          if login is None:
     598                  return False
     599
     600          if login.host is not None:
     601                  if login.host.strip() == '':
     602                          login.host = None
     603
     604          global _default_login
     605          _default_login = login
     606          _log.info('setting default login from [%s] to [%s]' % (_default_login, login))
     607
     608          dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password)
     609
     610          global _default_dsn
     611          if _default_dsn is None:
     612                  old_dsn = 'None'
     613          else:
     614                  old_dsn = regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, _default_dsn)
     615          _log.info ('setting default DSN from [%s] to [%s]',
     616                  old_dsn,
     617                  regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, dsn)
     618          )
     619          _default_dsn = dsn
     620
     621          return True
     622
     623  #------------------------------------------------------------------------
     624 -def log_auth_environment():
     625          try:
     626                  pgpass_file = os.path.expanduser(os.path.join('~', '.pgpass'))
     627                  if os.path.exists(pgpass_file):
     628                          _log.debug('standard .pgpass (%s) exists', pgpass_file)
     629                  else:
     630                          _log.debug('standard .pgpass (%s) not found', pgpass_file)
     631                  pgpass_var = os.getenv('PGPASSFILE')
     632                  if pgpass_var is None:
     633                          _log.debug('$PGPASSFILE not set')
     634                  else:
     635                          if os.path.exists(pgpass_var):
     636                                  _log.debug('$PGPASSFILE=%s exists', pgpass_var)
     637                          else:
     638                                  _log.debug('$PGPASSFILE=%s not found')
     639          except Exception:
     640                  _log.exception('cannot detect .pgpass and or $PGPASSFILE')
     641
     642  # =======================================================================
     643  # netadata API
     644  # =======================================================================
     645 -def database_schema_compatible(link_obj=None, version=None, verbose=True):
     646          expected_hash = known_schema_hashes[version]
     647          if version == 0:
     648                  args = {'ver': 9999}
     649          else:
     650                  args = {'ver': version}
     651          rows, idx = run_ro_queries (
     652                  link_obj = link_obj,
     653                  queries = [{
     654                          'cmd': 'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5',
     655                          'args': args
     656                  }]
     657          )
     658          if rows[0]['md5'] != expected_hash:
     659                  _log.error('database schema version mismatch')
     660                  _log.error('expected: %s (%s)' % (version, expected_hash))
     661                  _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5']))
     662                  if verbose:
     663                          _log.debug('schema dump follows:')
     664                          for line in get_schema_structure(link_obj = link_obj).split():
     665                                  _log.debug(line)
     666                          _log.debug('schema revision history dump follows:')
     667                          for line in get_schema_revision_history(link_obj = link_obj):
     668                                  _log.debug(' - '.join(line))
     669                  return False
     670          _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5']))
     671          return True
     672
     673  #------------------------------------------------------------------------
     674 -def get_schema_version(link_obj=None):
     675          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
     676          try:
     677                  return map_schema_hash2version[rows[0]['md5']]
     678          except KeyError:
     679                  return 'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
     680
     681  #------------------------------------------------------------------------
     682 -def get_schema_structure(link_obj=None):
     683          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select gm.concat_table_structure()'}])
     684          return rows[0][0]
     685
     686  #------------------------------------------------------------------------
     687 -def get_schema_hash(link_obj=None):
     688          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
     689          return rows[0]['md5']
     690
     691  #------------------------------------------------------------------------
     692 -def get_schema_revision_history(link_obj=None):
     693
     694          if table_exists(link_obj = link_obj, schema = 'gm', table = 'schema_revision'):
     695                  cmd = """
     696                          SELECT
     697                                  imported::text,
     698                                  version,
     699                                  filename
     700                          FROM gm.schema_revision
     701                          ORDER BY imported"""
     702          elif table_exists(link_obj = link_obj, schema = 'public', table = 'gm_schema_revision'):
     703                  cmd = """
     704                          SELECT
     705                                  imported::text,
     706                                  version,
     707                                  filename
     708                          FROM public.gm_schema_revision
     709                          ORDER BY imported"""
     710          else:
     711                  return []
     712
     713          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}])
     714          return rows
     715  #------------------------------------------------------------------------
     716 -def get_current_user():
     717          rows, idx = run_ro_queries(queries = [{'cmd': 'select CURRENT_USER'}])
     718          return rows[0][0]
     719
     720  #------------------------------------------------------------------------
     721 -def get_foreign_keys2column(schema='public', table=None, column=None, link_obj=None):
     722          """Get the foreign keys pointing to schema.table.column.
     723
     724          Does not properly work with multi-column FKs.
     725          GNUmed doesn't use any, however.
     726          """
     727          args = {
     728                  'schema': schema,
     729                  'tbl': table,
     730                  'col': column
     731          }
     732          cmd = """
     733  SELECT
     734          %(schema)s AS referenced_schema,
     735          %(tbl)s AS referenced_table,
     736          %(col)s AS referenced_column,
     737          pgc.confkey AS referenced_column_list,
     738
     739          pgc.conrelid::regclass AS referencing_table,
     740          pgc.conkey AS referencing_column_list,
     741          (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) AS referencing_column
     742  FROM
     743          pg_constraint pgc
     744  WHERE
     745          pgc.contype = 'f'
     746                  AND
     747          pgc.confrelid = (
     748                  select oid from pg_class where relname = %(tbl)s and relnamespace = (
     749                          select oid from pg_namespace where nspname = %(schema)s
     750                   )
     751          )       and
     752          (
     753                  select attnum
     754                  from pg_attribute
     755                  where
     756                          attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = (
     757                                  select oid from pg_namespace where nspname = %(schema)s
     758                          ))
     759                                  and
     760                          attname = %(col)s
     761          ) = any(pgc.confkey)
     762  """
     763          rows, idx = run_ro_queries (
     764                  link_obj = link_obj,
     765                  queries = [
     766                          {'cmd': cmd, 'args': args}
     767                  ]
     768          )
     769
     770          return rows
     771
     772  #------------------------------------------------------------------------
     773 -def get_index_name(indexed_table=None, indexed_column=None, link_obj=None):
     774
     775          args = {
     776                  'idx_tbl': indexed_table,
     777                  'idx_col': indexed_column
     778          }
     779          rows, idx = run_ro_queries (
     780                  link_obj = link_obj,
     781                  queries = [{'cmd': SQL_get_index_name, 'args': args}],
     782                  get_col_idx = False
     783          )
     784
     785          return rows
     786
     787  #------------------------------------------------------------------------
     788 -def get_foreign_key_names(src_schema=None, src_table=None, src_column=None, target_schema=None, target_table=None, target_column=None, link_obj=None):
     789
     790          args = {
     791                  'src_schema': src_schema,
     792                  'src_tbl': src_table,
     793                  'src_col': src_column,
     794                  'target_schema': target_schema,
     795                  'target_tbl': target_table,
     796                  'target_col': target_column
     797          }
     798
     799          rows, idx = run_ro_queries (
     800                  link_obj = link_obj,
     801                  queries = [{'cmd': SQL_foreign_key_name, 'args': args}],
     802                  get_col_idx = False
     803          )
     804
     805          return rows
     806
     807  #------------------------------------------------------------------------
     808 -def get_child_tables(schema='public', table=None, link_obj=None):
     809          """Return child tables of <table>."""
     810          cmd = """
     811  select
     812          pgn.nspname as namespace,
     813          pgc.relname as table
     814  from
     815          pg_namespace pgn,
     816          pg_class pgc
     817  where
     818          pgc.relnamespace = pgn.oid
     819                  and
     820          pgc.oid in (
     821                  select inhrelid from pg_inherits where inhparent = (
     822                          select oid from pg_class where
     823                                  relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
     824                                  relname = %(table)s
     825                  )
     826          )"""
     827          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}])
     828          return rows
     829
     830  #------------------------------------------------------------------------
     831 -def schema_exists(link_obj=None, schema='gm'):
     832          cmd = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = %(schema)s)"""
     833          args = {'schema': schema}
     834          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
     835          return rows[0][0]
     836
     837  #------------------------------------------------------------------------
     838 -def table_exists(link_obj=None, schema=None, table=None):
     839          """Returns false, true."""
     840          cmd = """
     841  select exists (
     842          select 1 from information_schema.tables
     843          where
     844                  table_schema = %s and
     845                  table_name = %s and
     846                  table_type = 'BASE TABLE'
     847  )"""
     848          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}])
     849          return rows[0][0]
     850
     851  #------------------------------------------------------------------------
     852 -def function_exists(link_obj=None, schema=None, function=None):
     853
     854          cmd = """
     855                  SELECT EXISTS (
     856                          SELECT 1 FROM pg_proc
     857                          WHERE proname = %(func)s AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = %(schema)s)
     858                  )
     859          """
     860          args = {
     861                  'func': function,
     862                  'schema': schema
     863          }
     864          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
     865          return rows[0][0]
     866
     867  #------------------------------------------------------------------------
     868 -def get_col_indices(cursor = None):
     869          if cursor.description is None:
     870                  _log.error('no result description available: unused cursor or last query did not select rows')
     871                  return None
     872          col_indices = {}
     873          col_index = 0
     874          for col_desc in cursor.description:
     875                  col_name = col_desc[0]
     876                  # a query like "select 1,2;" will return two columns of the same name !
     877                  # hence adjust to that, note, however, that dict-style access won't work
     878                  # on results of such queries ...
     879                  if col_name in col_indices:
     880                          col_name = '%s_%s' % (col_name, col_index)
     881                  col_indices[col_name] = col_index
     882                  col_index += 1
     883
     884          return col_indices
     885  #------------------------------------------------------------------------
     886 -def get_col_defs(link_obj=None, schema='public', table=None):
     887          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}])
     888          col_names = []
     889          col_type = {}
     890          for row in rows:
     891                  col_names.append(row[0])
     892                  # map array types
     893                  if row[1].startswith('_'):
     894                          col_type[row[0]] = row[1][1:] + '[]'
     895                  else:
     896                          col_type[row[0]] = row[1]
     897          col_defs = []
     898          col_defs.append(col_names)
     899          col_defs.append(col_type)
     900          return col_defs
     901  #------------------------------------------------------------------------
     902 -def get_col_names(link_obj=None, schema='public', table=None):
     903          """Return column attributes of table"""
     904          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}])
     905          cols = []
     906          for row in rows:
     907                  cols.append(row[0])
     908          return cols
     909
     910  #------------------------------------------------------------------------
     911  # i18n functions
     912  #------------------------------------------------------------------------
     913 -def export_translations_from_database(filename=None):
     914          tx_file = io.open(filename, mode = 'wt', encoding = 'utf8')
     915          tx_file.write('-- GNUmed database string translations exported %s\n' % gmDateTime.pydt_now_here().strftime('%Y-%m-%d %H:%M'))
     916          tx_file.write('-- - contains translations for each of [%s]\n' % ', '.join(get_translation_languages()))
     917          tx_file.write('-- - user database language is set to [%s]\n\n' % get_current_user_language())
     918          tx_file.write('-- Please email this file to <gnumed-devel@gnu.org>.\n')
     919          tx_file.write('-- ----------------------------------------------------------------------------------------------\n\n')
     920          tx_file.write('set default_transaction_read_only to off;\n\n')
     921          tx_file.write("set client_encoding to 'utf-8';\n\n")
     922          tx_file.write('\\unset ON_ERROR_STOP\n\n')
     923
     924          cmd = 'SELECT lang, orig, trans FROM i18n.translations ORDER BY lang, orig'
     925          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
     926          for row in rows:
     927                  line = "select i18n.upd_tx(E'%s', E'%s', E'%s');\n" % (
     928                          row['lang'].replace("'", "\\'"),
     929                          row['orig'].replace("'", "\\'"),
     930                          row['trans'].replace("'", "\\'")
     931                  )
     932                  tx_file.write(line)
     933          tx_file.write('\n')
     934
     935          tx_file.write('\set ON_ERROR_STOP 1\n')
     936          tx_file.close()
     937
     938          return True
     939
     940  #------------------------------------------------------------------------
     941 -def delete_translation_from_database(link_obj=None, language=None, original=None):
     942          cmd = 'DELETE FROM i18n.translations WHERE lang = %(lang)s AND orig = %(orig)s'
     943          args = {'lang': language, 'orig': original}
     944          run_rw_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}], return_data = False, end_tx = True)
     945          return True
     946
     947  #------------------------------------------------------------------------
     948 -def update_translation_in_database(language=None, original=None, translation=None, link_obj=None):
     949          if language is None:
     950                  cmd = 'SELECT i18n.upd_tx(%(orig)s, %(trans)s)'
     951          else:
     952                  cmd = 'SELECT i18n.upd_tx(%(lang)s, %(orig)s, %(trans)s)'
     953          args = {'lang': language, 'orig': original, 'trans': translation}
     954          run_rw_queries(queries = [{'cmd': cmd, 'args': args}], return_data = False, link_obj = link_obj)
     955          return args
     956
     957  #------------------------------------------------------------------------
     958 -def get_translation_languages():
     959          rows, idx = run_ro_queries (
     960                  queries = [{'cmd': 'select distinct lang from i18n.translations'}]
     961          )
     962          return [ r[0] for r in rows ]
     963
     964  #------------------------------------------------------------------------
     965 -def get_database_translations(language=None, order_by=None):
     966
     967          args = {'lang': language}
     968          _log.debug('language [%s]', language)
     969
     970          if order_by is None:
     971                  order_by = 'ORDER BY %s' % order_by
     972          else:
     973                  order_by = 'ORDER BY lang, orig'
     974
     975          if language is None:
     976                  cmd = """
     977                  SELECT DISTINCT ON (orig, lang)
     978                          lang, orig, trans
     979                  FROM ((
     980
     981                          -- strings stored as translation keys whether translated or not
     982                          SELECT
     983                                  NULL as lang,
     984                                  ik.orig,
     985                                  NULL AS trans
     986                          FROM
     987                                  i18n.keys ik
     988
     989                  ) UNION ALL (
     990
     991                          -- already translated strings
     992                          SELECT
     993                                  it.lang,
     994                                  it.orig,
     995                                  it.trans
     996                          FROM
     997                                  i18n.translations it
     998
     999                  )) as translatable_strings
    1000                  %s""" % order_by
    1001          else:
    1002                  cmd = """
    1003                  SELECT DISTINCT ON (orig, lang)
    1004                          lang, orig, trans
    1005                  FROM ((
    1006
    1007                          -- strings stored as translation keys whether translated or not
    1008                          SELECT
    1009                                  %%(lang)s as lang,
    1010                                  ik.orig,
    1011                                  i18n._(ik.orig, %%(lang)s) AS trans
    1012                          FROM
    1013                                  i18n.keys ik
    1014
    1015                  ) UNION ALL (
    1016
    1017                          -- already translated strings
    1018                          SELECT
    1019                                  %%(lang)s as lang,
    1020                                  it.orig,
    1021                                  i18n._(it.orig, %%(lang)s) AS trans
    1022                          FROM
    1023                                  i18n.translations it
    1024
    1025                  )) AS translatable_strings
    1026                  %s""" % order_by
    1027
    1028          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
    1029
    1030          if rows is None:
    1031                  _log.error('no translatable strings found')
    1032          else:
    1033                  _log.debug('%s translatable strings found', len(rows))
    1034
    1035          return rows
    1036
    1037  #------------------------------------------------------------------------
    1038 -def get_current_user_language():
    1039          cmd = 'select i18n.get_curr_lang()'
    1040          rows, idx = run_ro_queries(queries = [{'cmd': cmd}])
    1041          return rows[0][0]
    1042
    1043  #------------------------------------------------------------------------
    1044 -def set_user_language(user=None, language=None):
    1045          """Set the user language in the database.
    1046
    1047          user = None: current db user
    1048          language = None: unset
    1049          """
    1050          _log.info('setting database language for user [%s] to [%s]', user, language)
    1051
    1052          args = {
    1053                  'usr': user,
    1054                  'lang': language
    1055          }
    1056
    1057          if language is None:
    1058                  if user is None:
    1059                          queries = [{'cmd': 'select i18n.unset_curr_lang()'}]
    1060                  else:
    1061                          queries = [{'cmd': 'select i18n.unset_curr_lang(%(usr)s)', 'args': args}]
    1062                  queries.append({'cmd': 'select True'})
    1063          else:
    1064                  if user is None:
    1065                          queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s)', 'args': args}]
    1066                  else:
    1067                          queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}]
    1068
    1069          rows, idx = run_rw_queries(queries = queries, return_data = True)
    1070
    1071          if not rows[0][0]:
    1072                  _log.error('cannot set database language to [%s] for user [%s]', language, user)
    1073
    1074          return rows[0][0]
    1075  #------------------------------------------------------------------------
    1076 -def force_user_language(language=None):
    1077          """Set the user language in the database.
    1078
    1079          - regardless of whether there is any translation available.
    1080          - only for the current user
    1081          """
    1082          _log.info('forcing database language for current db user to [%s]', language)
    1083
    1084          run_rw_queries(queries = [{
    1085                  'cmd': 'select i18n.force_curr_lang(%(lang)s)',
    1086                  'args': {'lang': language}
    1087          }])
    1088
    1089  # =======================================================================
    1090  # query runners and helpers
    1091  # =======================================================================
    1092 -def send_maintenance_notification():
    1093          cmd = 'notify "db_maintenance_warning"'
    1094          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
    1095
    1096  #------------------------------------------------------------------------
    1097 -def send_maintenance_shutdown():
    1098          cmd = 'notify "db_maintenance_disconnect"'
    1099          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
    1100
    1101  #------------------------------------------------------------------------
    1102 -def is_pg_interval(candidate=None):
    1103          cmd = 'SELECT %(candidate)s::interval'
    1104          try:
    1105                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
    1106                  return True
    1107          except:
    1108                  cmd = 'SELECT %(candidate)s::text::interval'
    1109                  try:
    1110                          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
    1111                          return True
    1112                  except:
    1113                          return False
    1114
    1115  #------------------------------------------------------------------------
    1116 -def lock_row(link_obj=None, table=None, pk=None, exclusive=False):
    1117          """Uses pg_advisory(_shared).
    1118
    1119          - locks stack upon each other and need one unlock per lock
    1120          - same connection:
    1121                  - all locks succeed
    1122          - different connections:
    1123                  - shared + shared succeed
    1124                  - shared + exclusive fail
    1125          """
    1126          _log.debug('locking row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1127          if exclusive:
    1128                  cmd = """SELECT pg_try_advisory_lock('%s'::regclass::oid::int, %s)""" % (table, pk)
    1129          else:
    1130                  cmd = """SELECT pg_try_advisory_lock_shared('%s'::regclass::oid::int, %s)""" % (table, pk)
    1131          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
    1132          if rows[0][0]:
    1133                  return True
    1134          _log.warning('cannot lock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1135          return False
    1136
    1137  #------------------------------------------------------------------------
    1138 -def unlock_row(link_obj=None, table=None, pk=None, exclusive=False):
    1139          """Uses pg_advisory_unlock(_shared).
    1140
    1141          - each lock needs one unlock
    1142          """
    1143          _log.debug('trying to unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1144          if exclusive:
    1145                  cmd = "SELECT pg_advisory_unlock('%s'::regclass::oid::int, %s)" % (table, pk)
    1146          else:
    1147                  cmd = "SELECT pg_advisory_unlock_shared('%s'::regclass::oid::int, %s)" % (table, pk)
    1148          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
    1149          if rows[0][0]:
    1150                  return True
    1151          _log.warning('cannot unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1152          return False
    1153
    1154  #------------------------------------------------------------------------
    1155 -def row_is_locked(table=None, pk=None):
    1156          """Looks at pk_locks
    1157
    1158          - does not take into account locks other than 'advisory', however
    1159          """
    1160          cmd = """SELECT EXISTS (
    1161                  SELECT 1 FROM pg_locks WHERE
    1162                          classid = '%s'::regclass::oid::int
    1163                                  AND
    1164                          objid = %s
    1165                                  AND
    1166                          locktype = 'advisory'
    1167          )""" % (table, pk)
    1168          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
    1169          if rows[0][0]:
    1170                  _log.debug('row is locked: [%s] [%s]', table, pk)
    1171                  return True
    1172          _log.debug('row is NOT locked: [%s] [%s]', table, pk)
    1173          return False
    1174
    1175  #------------------------------------------------------------------------
    1176  # BYTEA cache handling
    1177  #------------------------------------------------------------------------
    1178  #_BYTEA_CACHE_SUBDIR = None
    1179  #
    1180  #def __get_bytea_cache_dir():
    1181  #       """Get, and create if necessary, the BYTEA cache directory.
    1182  #
    1183  #       The BYTEA cache needs to be tied to the database login
    1184  #       because different logins might have access to different
    1185  #       data even with the same query.
    1186  #       """
    1187  #       global _BYTEA_CACHE_SUBDIR
    1188  #       if _BYTEA_CACHE_SUBDIR is not None:
    1189  #               return _BYTEA_CACHE_SUBDIR
    1190  #
    1191  #       subdir_key = '%s::%s::%s::%s' % (
    1192  #               _default_login.host,
    1193  #               _default_login.port,
    1194  #               _default_login.database,
    1195  #               _default_login.user
    1196  #       )
    1197  #       md5 = hashlib.md5()
    1198  #       md5.update(subdir_key.encode('utf8'))
    1199  #       md5_sum = md5.hexdigest()
    1200  #       subdir = os.path.join(gmTools.gmPaths().bytea_cache_dir, md5_sum)
    1201  #       gmTools.mkdir(subdir, 0o700)
    1202  #       _log.info('BYTEA cache dir for [%s]: %s', subdir_key, subdir)
    1203  #       _BYTEA_CACHE_SUBDIR = subdir
    1204  #       return _BYTEA_CACHE_SUBDIR
    1205  #
    1206  #------------------------------------------------------------------------
    1207 -def __generate_cached_filename(cache_key_data):
    1208          md5 = hashlib.md5()
    1209          md5.update(('%s' % cache_key_data).encode('utf8'))
    1210          return os.path.join(gmTools.gmPaths().bytea_cache_dir, md5.hexdigest())
    1211
    1212  #------------------------------------------------------------------------
    1213 -def __store_file_in_cache(filename, cache_key_data):
    1214          cached_name = __generate_cached_filename(cache_key_data)
    1215          _log.debug('[%s] -> [%s] -> [%s]', filename, cache_key_data, cached_name)
    1216          if not gmTools.remove_file(cached_name, log_error = True, force = True):
    1217                  _log.error('cannot remove existing file [%s] for key [%s] from cache', filename, cached_name)
    1218                  return None
    1219
    1220          PERMS_owner_only = 0o0660
    1221          try:
    1222                  shutil.copyfile(filename, cached_name, follow_symlinks = True)
    1223                  os.chmod(cached_name, PERMS_owner_only)
    1224          except shutil.SameFileError:
    1225                  _log.exception('file seems to exist in cache, despite having checked and possible removed it just before')
    1226                  # don't use that file, it is unsafe, it might have come from
    1227                  # a race being exploited to make us use the wrong data, this
    1228                  # then constitutes a DOS attack against the cache but that's
    1229                  # far less problematic than using the wrong data for care
    1230                  return None
    1231          except OSError:
    1232                  _log.exception('cannot copy file into cache: [%s] -> [%s]', filename, cached_name)
    1233                  return None
    1234          except PermissionError:
    1235                  _log.exception('cannot set cache file [%s] permissions to [%s]', cached_name, stat.filemode(PERMS_owner_only))
    1236                  return None
    1237
    1238          return cached_name
    1239
    1240  #------------------------------------------------------------------------
    1241 -def __get_filename_in_cache(cache_key_data=None, data_size=None):
    1242          """Calculate and verify filename in cache given cache key details."""
    1243          cached_name = __generate_cached_filename(cache_key_data)
    1244          try:
    1245                  stat = os.stat(cached_name)
    1246          except FileNotFoundError:
    1247                  return None
    1248          _log.debug('cache hit: [%s] -> [%s] (%s)', cache_key_data, cached_name, stat)
    1249          if os.path.islink(cached_name) or (not os.path.isfile(cached_name)):
    1250                  _log.error('object in cache is not a regular file: %s', cached_name)
    1251                  _log.error('possibly an attack, removing')
    1252                  if gmTools.remove_file(cached_name, log_error = True):
    1253                          return None
    1254                  raise Exception('cannot delete suspicious object in cache dir: %s', cached_name)
    1255          if stat.st_size == data_size:
    1256                  return cached_name
    1257          _log.debug('size in cache [%s] <> expected size [%s], removing cached file', stat.st_size, data_size)
    1258          if gmTools.remove_file(cached_name, log_error = True):
    1259                  return None
    1260          raise Exception('cannot remove suspicous object from cache dir: %s', cached_name)
    1261
    1262  #------------------------------------------------------------------------
    1263 -def __get_file_from_cache(filename, cache_key_data=None, data_size=None, link2cached=True):
    1264          """Get file from cache if available."""
    1265          cached_filename = __get_filename_in_cache(cache_key_data = cache_key_data, data_size = data_size)
    1266          if cached_filename is None:
    1267                  return False
    1268          if link2cached:
    1269                  try:
    1270                          # (hard)link as desired name, quite a few security
    1271                          # and archival tools refuse to handle softlinks
    1272                          os.link(cached_filename, filename)
    1273                          _log.debug('hardlinked [%s] as [%s]', cached_filename, filename)
    1274                          return True
    1275                  except Exception:
    1276                          pass
    1277                  _log.debug('cannot hardlink to cache, trying copy-from-cache')
    1278          # copy from cache
    1279          try:
    1280                  shutil.copyfile(cached_filename, filename, follow_symlinks = True)
    1281                  return True
    1282          except shutil.SameFileError:
    1283                  # flaky - might be same name but different content
    1284                  pass
    1285          except OSError:
    1286                  _log.exception('cannot copy cached file [%s] into [%s]', cached_filename, filename)
    1287          # if cache fails entirely -> fall through to new file
    1288          _log.debug('downloading new copy of file, despite found in cache')
    1289          return False
    1290
    1291  #------------------------------------------------------------------------
    1292 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None, conn=None, link2cached=True):
    1293
    1294          if data_size is None:
    1295                  rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
    1296                  data_size = rows[0][0]
    1297                  if data_size in [None, 0]:
    1298                          conn.rollback()
    1299                          # should an empty file be created if size == 0 ?
    1300                          return True
    1301
    1302          # actually needs to get values from <conn> or "default conn" if <conn> is None
    1303          cache_key_data = '<%s>@%s:%s/%s::%s' % (
    1304                  _default_login.user,
    1305                  _default_login.host,
    1306                  _default_login.port,
    1307                  _default_login.database,
    1308                  data_query
    1309          )
    1310          found_in_cache = __get_file_from_cache(filename, cache_key_data = cache_key_data, data_size = data_size, link2cached = link2cached)
    1311          if found_in_cache:
    1312                  return True
    1313
    1314          outfile = io.open(filename, 'wb')
    1315          result = bytea2file_object (
    1316                  data_query = data_query,
    1317                  file_obj = outfile,
    1318                  chunk_size = chunk_size,
    1319                  data_size = data_size,
    1320                  data_size_query = data_size_query,
    1321                  conn = conn
    1322          )
    1323          outfile.close()
    1324          __store_file_in_cache(filename, cache_key_data)
    1325          return result
    1326
    1327  #------------------------------------------------------------------------
    1328 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None, conn=None):
    1329          """Store data from a bytea field into a file.
    1330
    1331          <data_query>
    1332          - dict {'cmd': ..., 'args': ...}
    1333          - 'cmd' must be a string containing "... substring(data from %(start)s for %(size)s) ..."
    1334          - 'args' must be a dict
    1335          - must return one row with one field of type bytea
    1336          <file>
    1337          - must be a file like Python object
    1338          <data_size>
    1339          - integer of the total size of the expected data or None
    1340          <data_size_query>
    1341          - dict {'cmd': ..., 'args': ...}
    1342          - must return one row with one field with the octet_length() of the data field
    1343          - used only when <data_size> is None
    1344          """
    1345          if data_size == 0:
    1346                  return True
    1347
    1348          # If the client sets an encoding other than the default we
    1349          # will receive encoding-parsed data which isn't the binary
    1350          # content we want. Hence we need to get our own connection.
    1351          # It must be a read-write one so that we don't affect the
    1352          # encoding for other users of the shared read-only
    1353          # connections.
    1354          # Actually, encodings shouldn't be applied to binary data
    1355          # (eg. bytea types) in the first place but that is only
    1356          # reported to be fixed > v7.4.
    1357          # further tests reveal that at least on PG 8.0 this bug still
    1358          # manifests itself
    1359          if conn is None:
    1360                  conn = get_raw_connection(readonly = True)
    1361
    1362          if data_size is None:
    1363                  rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
    1364                  data_size = rows[0][0]
    1365                  if data_size in [None, 0]:
    1366                          conn.rollback()
    1367                          return True
    1368
    1369          max_chunk_size = 1024 * 1024 * 20                       # 20 MB, works for typical CR DICOMs
    1370          if chunk_size == 0:
    1371                  chunk_size = min(data_size, max_chunk_size)
    1372
    1373          _log.debug('expecting %s bytes of BYTEA data in chunks of %s bytes', data_size, chunk_size)
    1374
    1375          # Windoze sucks: it can't transfer objects of arbitrary size,
    1376          # anyways, we need to split the transfer,
    1377          # however, only possible if postgres >= 7.2
    1378          needed_chunks, remainder = divmod(data_size, chunk_size)
    1379          _log.debug('# of chunks: %s; remainder: %s bytes', needed_chunks, remainder)
    1380
    1381          # retrieve chunks, skipped if data size < chunk size,
    1382          # does this not carry the danger of cutting up multi-byte escape sequences ?
    1383          # no, since bytea is binary,
    1384          # yes, since in bytea there are *some* escaped values, still
    1385          # no, since those are only escaped during *transfer*, not on-disk, hence
    1386          # only complete escape sequences are put on the wire
    1387          for chunk_id in range(needed_chunks):
    1388                  chunk_start = (chunk_id * chunk_size) + 1
    1389                  data_query['args']['start'] = chunk_start
    1390                  data_query['args']['size'] = chunk_size
    1391                  try:
    1392                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1393                  except:
    1394                          _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size))
    1395                          conn.rollback()
    1396                          raise
    1397                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1398                  file_obj.write(rows[0][0])
    1399
    1400          # retrieve remainder
    1401          if remainder > 0:
    1402                  chunk_start = (needed_chunks * chunk_size) + 1
    1403                  data_query['args']['start'] = chunk_start
    1404                  data_query['args']['size'] = remainder
    1405                  try:
    1406                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1407                  except:
    1408                          _log.error('cannot retrieve remaining [%s] bytes' % remainder)
    1409                          conn.rollback()
    1410                          raise
    1411                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1412                  file_obj.write(rows[0][0])
    1413
    1414          conn.rollback()
    1415          return True
    1416
    1417  #------------------------------------------------------------------------
    1418 -def file2bytea(query=None, filename=None, args=None, conn=None, file_md5=None):
    1419          """Store data from a file into a bytea field.
    1420
    1421          The query must:
    1422          - be in unicode
    1423          - contain a format spec identifying the row (eg a primary key)
    1424            matching <args> if it is an UPDATE
    1425          - contain a format spec " <field> = %(data)s::bytea"
    1426
    1427          The query CAN return the MD5 of the inserted data:
    1428                  RETURNING md5(<field>) AS md5
    1429          in which case it will compare it to the md5
    1430          of the file.
    1431          """
    1432          # read data from file
    1433          infile = open(filename, "rb")
    1434          data_as_byte_string = infile.read()
    1435          infile.close()
    1436          if args is None:
    1437                  args = {}
    1438          # really still needed for BYTEA input ?
    1439          args['data'] = memoryview(data_as_byte_string)
    1440          del(data_as_byte_string)
    1441
    1442          # insert the data
    1443          if conn is None:
    1444                  conn = get_raw_connection(readonly = False)
    1445                  close_conn = True
    1446          else:
    1447                  close_conn = False
    1448
    1449          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = (file_md5 is not None))
    1450
    1451          success_status = True
    1452          if file_md5 is None:
    1453                  conn.commit()
    1454          else:
    1455                  db_md5 = rows[0]['md5']
    1456                  if file_md5 != db_md5:
    1457                          conn.rollback()
    1458                          success_status = False
    1459                          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1460                  else:
    1461                          conn.commit()
    1462                          _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1463
    1464          if close_conn:
    1465                  conn.close()
    1466
    1467          return success_status
    1468
    1469  #------------------------------------------------------------------------
    1470 -def file2lo(filename=None, conn=None, check_md5=False):
    1471          # 1 GB limit unless 64 bit Python build ...
    1472          file_size = os.path.getsize(filename)
    1473          if file_size > (1024 * 1024) * 1024:
    1474                  _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
    1475  #               return -1
    1476
    1477          if conn is None:
    1478                  conn = get_raw_connection(readonly = False)
    1479                  close_conn = conn.close
    1480          else:
    1481                  close_conn = __noop
    1482          _log.debug('[%s] -> large object', filename)
    1483
    1484          # insert the data
    1485          lo = conn.lobject(0, 'w', 0, filename)
    1486          lo_oid = lo.oid
    1487          lo.close()
    1488          _log.debug('large object OID: %s', lo_oid)
    1489
    1490          # verify
    1491          if file_md5 is None:
    1492                  conn.commit()
    1493                  close_conn()
    1494                  return lo_oid
    1495          cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
    1496          args = {'loid': lo_oid}
    1497          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
    1498          db_md5 = rows[0][0]
    1499          if file_md5 == db_md5:
    1500                  conn.commit()
    1501                  close_conn()
    1502                  _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1503                  return lo_oid
    1504          conn.rollback()
    1505          close_conn()
    1506          _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
    1507          return -1
    1508
    1509  #------------------------------------------------------------------------
    1510 -def file2bytea_lo(filename=None, conn=None, file_md5=None):
    1511          # 1 GB limit unless 64 bit Python build ...
    1512          file_size = os.path.getsize(filename)
    1513          if file_size > (1024 * 1024) * 1024:
    1514                  _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
    1515  #               return -1
    1516
    1517          if conn is None:
    1518                  conn = get_raw_connection(readonly = False)
    1519                  close_conn = conn.close
    1520          else:
    1521                  close_conn = __noop
    1522          _log.debug('[%s] -> large object', filename)
    1523
    1524          # insert the data
    1525          lo = conn.lobject(0, 'w', 0, filename)
    1526          lo_oid = lo.oid
    1527          lo.close()
    1528          _log.debug('large object OID: %s', lo_oid)
    1529
    1530          # verify
    1531          if file_md5 is None:
    1532                  conn.commit()
    1533                  close_conn()
    1534                  return lo_oid
    1535          cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
    1536          args = {'loid': lo_oid}
    1537          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
    1538          db_md5 = rows[0][0]
    1539          if file_md5 == db_md5:
    1540                  conn.commit()
    1541                  close_conn()
    1542                  _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1543                  return lo_oid
    1544          conn.rollback()
    1545          close_conn()
    1546          _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
    1547          return -1
    1548
    1549  #------------------------------------------------------------------------
    1550 -def file2bytea_copy_from(table=None, columns=None, filename=None, conn=None, md5_query=None, file_md5=None):
    1551          # md5_query: dict{'cmd': ..., 'args': ...}
    1552
    1553          # UNTESTED
    1554
    1555          chunk_size = 32 * (1024 * 1024)
    1556          _log.debug('[%s] (%s bytes) --(%s bytes)-> %s(%s)', filename, os.path.getsize(filename), chunk_size, table, columns)
    1557          if conn is None:
    1558                  conn = get_raw_connection(readonly = False)
    1559                  close_conn = True
    1560          else:
    1561                  close_conn = False
    1562          curs = conn.cursor()
    1563          # write
    1564          infile = open(filename, "rb")
    1565          curs.copy_from(infile, table, size = chunk_size, columns = columns)
    1566          infile.close()
    1567          curs.close()
    1568          if None in [file_md5, md5_query]:
    1569                  conn.commit()
    1570                  close_conn()
    1571                  return True
    1572          # verify
    1573          rows, idx = run_ro_queries(link_obj = conn, queries = [md5_query])
    1574          db_md5 = rows[0][0]
    1575          if file_md5 == db_md5:
    1576                  conn.commit()
    1577                  close_conn()
    1578                  _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1579                  return True
    1580          close_conn()
    1581          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1582          return False
    1583
    1584  #------------------------------------------------------------------------
    1585 -def file2bytea_overlay(query=None, args=None, filename=None, conn=None, md5_query=None, file_md5=None):
    1586          """Store data from a file into a bytea field.
    1587
    1588          The query must:
    1589          - 'cmd' must be in unicode
    1590          - 'cmd' must contain a format spec identifying the row (eg
    1591            a primary key) matching <args> if it is an UPDATE
    1592          - 'cmd' must contain "... SET ... <some_bytea_field> = OVERLAY(some_bytea_field PLACING %(data)s::bytea FROM %(start)s FOR %(size)s) ..."
    1593          - 'args' must be a dict matching 'cmd'
    1594
    1595          The query CAN return the MD5 of the inserted data:
    1596                  RETURNING md5(<field>) AS md5
    1597          in which case it will compare it to the md5
    1598          of the file.
    1599
    1600          UPDATE
    1601                  the_table
    1602          SET
    1603                  bytea_field = OVERLAY (
    1604                          coalesce(bytea_field, '':bytea),
    1605                          PLACING
    1606                                  %(data)s::bytea
    1607                          FROM
    1608                                  %(start)s
    1609                          FOR
    1610                                  %(size)s
    1611                  )
    1612          WHERE
    1613                  primary_key = pk_value
    1614
    1615          SELECT md5(bytea_field) FROM the_table WHERE primary_key = pk_value
    1616          """
    1617          chunk_size = 32 * (1024 * 1024)
    1618          file_size = os.path.getsize(filename)
    1619          if file_size <= chunk_size:
    1620                  chunk_size = file_size
    1621          needed_chunks, remainder = divmod(file_size, chunk_size)
    1622          _log.debug('file data: %s bytes, chunks: %s, chunk size: %s bytes, remainder: %s bytes', file_size, needed_chunks, chunk_size, remainder)
    1623
    1624          if conn is None:
    1625                  conn = get_raw_connection(readonly = False)
    1626                  close_conn = conn.close
    1627          else:
    1628                  close_conn = __noop
    1629
    1630          infile = open(filename, "rb")
    1631          # write chunks
    1632          for chunk_id in range(needed_chunks):
    1633                  chunk_start = (chunk_id * chunk_size) + 1
    1634                  args['start'] = chunk_start
    1635                  args['size'] = chunk_size
    1636                  data_as_byte_string = infile.read(chunk_size)
    1637                  # really still needed for BYTEA input ?
    1638                  args['data'] = memoryview(data_as_byte_string)
    1639                  del(data_as_byte_string)
    1640                  try:
    1641                          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
    1642                  except Exception:
    1643                          _log.exception('cannot write chunk [%s/%s] of size [%s], try decreasing chunk size', chunk_id+1, needed_chunks, chunk_size)
    1644                          conn.rollback()
    1645                          close_conn()
    1646                          infile.close()
    1647                          raise
    1648          # write remainder
    1649          if remainder > 0:
    1650                  chunk_start = (needed_chunks * chunk_size) + 1
    1651                  args['start'] = chunk_start
    1652                  args['size'] = remainder
    1653                  data_as_byte_string = infile.read(remainder)
    1654                  # really still needed for BYTEA input ?
    1655                  args['data'] = memoryview(data_as_byte_string)
    1656                  del(data_as_byte_string)
    1657                  try:
    1658                          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
    1659                  except Exception:
    1660                          _log.error('cannot retrieve remaining [%s] bytes' % remainder)
    1661                          conn.rollback()
    1662                          close_conn()
    1663                          infile.close()
    1664                          raise
    1665          infile.close()
    1666          if None in [file_md5, md5_query]:
    1667                  conn.commit()
    1668                  close_conn()
    1669                  return True
    1670          # verify
    1671          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': md5_query, 'args': args}])
    1672          db_md5 = rows[0][0]
    1673          if file_md5 == db_md5:
    1674                  conn.commit()
    1675                  close_conn()
    1676                  _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1677                  return True
    1678          close_conn()
    1679          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1680          return False
    1681
    1682  #---------------------------------------------------------------------------
    1683 -def run_sql_script(sql_script, conn=None):
    1684
    1685          if conn is None:
    1686                  conn = get_connection(readonly = False)
    1687
    1688          from Gnumed.pycommon import gmPsql
    1689          psql = gmPsql.Psql(conn)
    1690
    1691          if psql.run(sql_script) == 0:
    1692                  query = {
    1693                          'cmd': 'select gm.log_script_insertion(%(name)s, %(ver)s)',
    1694                          'args': {'name': sql_script, 'ver': 'current'}
    1695                  }
    1696                  run_rw_queries(link_obj = conn, queries = [query])
    1697                  conn.commit()
    1698                  return True
    1699
    1700          _log.error('error running sql script: %s', sql_script)
    1701          return False
    1702
    1703  #------------------------------------------------------------------------
    1704 -def sanitize_pg_regex(expression=None, escape_all=False):
    1705          """Escape input for use in a PostgreSQL regular expression.
    1706
    1707          If a fragment comes from user input and is to be used
    1708          as a regular expression we need to make sure it doesn't
    1709          contain invalid regex patterns such as unbalanced ('s.
    1710
    1711          <escape_all>
    1712                  True: try to escape *all* metacharacters
    1713                  False: only escape those which render the regex invalid
    1714          """
    1715          return expression.replace (
    1716                          '(', '\('
    1717                  ).replace (
    1718                          ')', '\)'
    1719                  ).replace (
    1720                          '[', '\['
    1721                  ).replace (
    1722                          '+', '\+'
    1723                  ).replace (
    1724                          '.', '\.'
    1725                  ).replace (
    1726                          '*', '\*'
    1727                  )
    1728                  #']', '\]',                     # not needed
    1729
    1730  #------------------------------------------------------------------------
    1731 -def capture_conn_state(conn=None):
    1732
    1733          tx_status = conn.get_transaction_status()
    1734          if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
    1735                  isolation_level = '%s (tx aborted or unknown, cannot retrieve)' % conn.isolation_level
    1736          else:
    1737                  isolation_level = '%s (%s)' % (conn.isolation_level, map_psyco_iso_level2str[conn.isolation_level])
    1738          conn_status = '%s (%s)' % (conn.status, map_psyco_conn_status2str[conn.status])
    1739          if conn.closed != 0:
    1740                  conn_status = 'undefined (%s)' % conn_status
    1741                  backend_pid = '<conn closed, cannot retrieve>'
    1742          else:
    1743                  backend_pid = conn.get_backend_pid()
    1744          try:
    1745                  conn_deferrable = conn.deferrable
    1746          except AttributeError:
    1747                  conn_deferrable = '<unavailable>'
    1748
    1749          d = {
    1750                  'identity': id(conn),
    1751                  'backend PID': backend_pid,
    1752                  'protocol version': conn.protocol_version,
    1753                  'encoding': conn.encoding,
    1754                  'closed': conn.closed,
    1755                  'readonly': conn.readonly,
    1756                  'autocommit': conn.autocommit,
    1757                  'isolation level (psyco)': isolation_level,
    1758                  'async': conn.async_,
    1759                  'deferrable': conn_deferrable,
    1760                  'transaction status': '%s (%s)' % (tx_status, map_psyco_tx_status2str[tx_status]),
    1761                  'connection status': conn_status,
    1762                  'executing async op': conn.isexecuting(),
    1763                  'type': type(conn)
    1764          }
    1765          return '%s\n' % conn + format_dict_like (
    1766                  d,
    1767                  relevant_keys = [
    1768                          'type',
    1769                          'identity',
    1770                          'backend PID',
    1771                          'protocol version',
    1772                          'encoding',
    1773                          'isolation level (psyco)',
    1774                          'readonly',
    1775                          'autocommit',
    1776                          'closed',
    1777                          'connection status',
    1778                          'transaction status',
    1779                          'deferrable',
    1780                          'async',
    1781                          'executing async op'
    1782                  ],
    1783                  tabular = True,
    1784                  value_delimiters = None
    1785          )
    1786
    1787  #------------------------------------------------------------------------
    1788 -def capture_cursor_state(cursor=None):
    1789          conn = cursor.connection
    1790
    1791          tx_status = conn.get_transaction_status()
    1792          if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
    1793                  isolation_level = '<tx aborted or unknown, cannot retrieve>'
    1794          else:
    1795                  isolation_level = conn.isolation_level
    1796          try:
    1797                  conn_deferrable = conn.deferrable
    1798          except AttributeError:
    1799                  conn_deferrable = '<unavailable>'
    1800
    1801          if cursor.query is None:
    1802                  query = '<no query>'
    1803          else:
    1804                  query = cursor.query.decode(errors = 'replace')
    1805
    1806          if conn.closed != 0:
    1807                  backend_pid = '<conn closed, cannot retrieve>'
    1808          else:
    1809                  backend_pid = conn.get_backend_pid()
    1810
    1811          txt = """Link state:
    1812  Cursor
    1813    identity: %s; name: %s
    1814    closed: %s; scrollable: %s; with hold: %s; arraysize: %s; itersize: %s;
    1815    last rowcount: %s; rownumber: %s; lastrowid (OID): %s;
    1816    last description: %s
    1817    statusmessage: %s
    1818  Connection
    1819    identity: %s; backend pid: %s; protocol version: %s;
    1820    closed: %s; autocommit: %s; isolation level: %s; encoding: %s; async: %s; deferrable: %s; readonly: %s;
    1821    TX status: %s; CX status: %s; executing async op: %s;
    1822  Query
    1823    %s
    1824  """ % (
    1825                  id(cursor),
    1826                  cursor.name,
    1827                  cursor.closed,
    1828                  cursor.scrollable,
    1829                  cursor.withhold,
    1830                  cursor.arraysize,
    1831                  cursor.itersize,
    1832                  cursor.rowcount,
    1833                  cursor.rownumber,
    1834                  cursor.lastrowid,
    1835                  cursor.description,
    1836                  cursor.statusmessage,
    1837
    1838                  id(conn),
    1839                  backend_pid,
    1840                  conn.protocol_version,
    1841                  conn.closed,
    1842                  conn.autocommit,
    1843                  isolation_level,
    1844                  conn.encoding,
    1845                  conn.async_,
    1846                  conn_deferrable,
    1847                  conn.readonly,
    1848                  map_psyco_tx_status2str[tx_status],
    1849                  map_psyco_conn_status2str[conn.status],
    1850                  conn.isexecuting(),
    1851
    1852                  query
    1853          )
    1854          return txt
    1855
    1856  #------------------------------------------------------------------------
    1857 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
    1858          """Run read-only queries.
    1859
    1860          <queries> must be a list of dicts:
    1861                  [
    1862                          {'cmd': <string>, 'args': <dict> or <tuple>},
    1863                          {...},
    1864                          ...
    1865                  ]
    1866          """
    1867          if isinstance(link_obj, dbapi._psycopg.cursor):
    1868                  curs = link_obj
    1869                  curs_close = lambda :1
    1870                  tx_rollback = lambda :1
    1871                  readonly_rollback_just_in_case = lambda :1
    1872          elif isinstance(link_obj, dbapi._psycopg.connection):
    1873                  curs = link_obj.cursor()
    1874                  curs_close = curs.close
    1875                  tx_rollback = link_obj.rollback
    1876                  if link_obj.autocommit is True:         # readonly connection ?
    1877                          readonly_rollback_just_in_case = link_obj.rollback
    1878                  else:
    1879                          # do not rollback readonly queries on passed-in readwrite
    1880                          # connections just in case because they may have already
    1881                          # seen fully legitimate write action which would get lost
    1882                          readonly_rollback_just_in_case = lambda :1
    1883          elif link_obj is None:
    1884                  conn = get_connection(readonly = True, verbose = verbose)
    1885                  curs = conn.cursor()
    1886                  curs_close = curs.close
    1887                  tx_rollback = conn.rollback
    1888                  readonly_rollback_just_in_case = conn.rollback
    1889          else:
    1890                  raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
    1891
    1892          if verbose:
    1893                  _log.debug('cursor: %s', curs)
    1894
    1895          for query in queries:
    1896                  try:
    1897                          args = query['args']
    1898                  except KeyError:
    1899                          args = None
    1900                  try:
    1901                          curs.execute(query['cmd'], args)
    1902                          if verbose:
    1903                                  _log.debug(capture_cursor_state(curs))
    1904                  except PG_ERROR_EXCEPTION as pg_exc:
    1905                          _log.error('query failed in RO connection')
    1906                          log_pg_exception_details(pg_exc)
    1907                          try:
    1908                                  curs_close()
    1909                          except PG_ERROR_EXCEPTION as pg_exc2:
    1910                                  _log.exception('cannot close cursor')
    1911                                  log_pg_exception_details(pg_exc2)
    1912                          try:
    1913                                  tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1914                          except PG_ERROR_EXCEPTION as pg_exc2:
    1915                                  _log.exception('cannot rollback transaction')
    1916                                  log_pg_exception_details(pg_exc2)
    1917                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    1918                                  details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
    1919                                  if curs.statusmessage != '':
    1920                                          details = 'Status: %s\n%s' % (
    1921                                                  curs.statusmessage.strip().strip('\n').strip().strip('\n'),
    1922                                                  details
    1923                                          )
    1924                                  if pg_exc.pgerror is None:
    1925                                          msg = '[%s]' % pg_exc.pgcode
    1926                                  else:
    1927                                          msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
    1928                                  raise gmExceptions.AccessDenied (
    1929                                          msg,
    1930                                          source = 'PostgreSQL',
    1931                                          code = pg_exc.pgcode,
    1932                                          details = details
    1933                                  )
    1934                          raise
    1935                  except Exception:
    1936                          _log.exception('error during query run in RO connection')
    1937                          _log.error(capture_cursor_state(curs))
    1938                          try:
    1939                                  curs_close()
    1940                          except PG_ERROR_EXCEPTION as pg_exc:
    1941                                  _log.exception('cannot close cursor')
    1942                                  log_pg_exception_details(pg_exc)
    1943                          try:
    1944                                  tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1945                          except PG_ERROR_EXCEPTION as pg_exc:
    1946                                  _log.exception('cannot rollback transation')
    1947                                  log_pg_exception_details(pg_exc)
    1948                          raise
    1949
    1950          data = None
    1951          col_idx = None
    1952          if return_data:
    1953                  data = curs.fetchall()
    1954                  if verbose:
    1955                          _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data))
    1956                          _log.debug('cursor description: %s', curs.description)
    1957                  if get_col_idx:
    1958                          col_idx = get_col_indices(curs)
    1959
    1960          curs_close()
    1961          # so we can see data committed meanwhile if the
    1962          # link object had been passed in and thusly might
    1963          # be part of a long-running read-only transaction
    1964          readonly_rollback_just_in_case()
    1965          return (data, col_idx)
    1966
    1967  #------------------------------------------------------------------------
    1968 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
    1969          """Convenience function for running a transaction
    1970             that is supposed to get committed.
    1971
    1972          <link_obj>
    1973                  can be either:
    1974                  - a cursor
    1975                  - a connection
    1976
    1977          <queries>
    1978                  is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>)
    1979                  to be executed as a single transaction, the last
    1980                  query may usefully return rows (such as a
    1981                  "SELECT currval('some_sequence')" statement)
    1982
    1983          <end_tx>
    1984                  - controls whether the transaction is finalized (eg.
    1985                    committed/rolled back) or not, this allows the
    1986                    call to run_rw_queries() to be part of a framing
    1987                    transaction
    1988                  - if link_obj is a connection then <end_tx> will
    1989                    default to False unless it is explicitly set to
    1990                    True which is taken to mean "yes, you do have full
    1991                    control over the transaction" in which case the
    1992                    transaction is properly finalized
    1993                  - if link_obj is a cursor we CANNOT finalize the
    1994                    transaction because we would need the connection for that
    1995                  - if link_obj is None <end_tx> will, of course, always be True
    1996
    1997          <return_data>
    1998                  - if true, the returned data will include the rows
    1999                    the last query selected
    2000                  - if false, it returns None instead
    2001
    2002          <get_col_idx>
    2003                  - if true, the returned data will include a dictionary
    2004                    mapping field names to column positions
    2005                  - if false, the returned data returns None instead
    2006
    2007          method result:
    2008                  - returns a tuple (data, idx)
    2009                  - <data>:
    2010                          * (None, None) if last query did not return rows
    2011                          * ("fetchall() result", <index>) if last query returned any rows
    2012                          * for <index> see <get_col_idx>
    2013          """
    2014          if isinstance(link_obj, dbapi._psycopg.cursor):
    2015                  conn_close = lambda :1
    2016                  conn_commit = lambda :1
    2017                  tx_rollback = lambda :1
    2018                  curs = link_obj
    2019                  curs_close = lambda :1
    2020                  notices_accessor = curs.connection
    2021          elif isinstance(link_obj, dbapi._psycopg.connection):
    2022                  conn_close = lambda :1
    2023                  if end_tx:
    2024                          conn_commit = link_obj.commit
    2025                          tx_rollback = link_obj.rollback
    2026                  else:
    2027                          conn_commit = lambda :1
    2028                          tx_rollback = lambda :1
    2029                  curs = link_obj.cursor()
    2030                  curs_close = curs.close
    2031                  notices_accessor = link_obj
    2032          elif link_obj is None:
    2033                  conn = get_connection(readonly=False)
    2034                  conn_close = conn.close
    2035                  conn_commit = conn.commit
    2036                  tx_rollback = conn.rollback
    2037                  curs = conn.cursor()
    2038                  curs_close = curs.close
    2039                  notices_accessor = conn
    2040          else:
    2041                  raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
    2042
    2043          for query in queries:
    2044                  try:
    2045                          args = query['args']
    2046                  except KeyError:
    2047                          args = None
    2048                  try:
    2049                          curs.execute(query['cmd'], args)
    2050                          if verbose:
    2051                                  _log.debug(capture_cursor_state(curs))
    2052                          for notice in notices_accessor.notices:
    2053                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    2054                          del notices_accessor.notices[:]
    2055                  # DB related exceptions
    2056                  except dbapi.Error as pg_exc:
    2057                          _log.error('query failed in RW connection')
    2058                          log_pg_exception_details(pg_exc)
    2059                          for notice in notices_accessor.notices:
    2060                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    2061                          del notices_accessor.notices[:]
    2062                          try:
    2063                                  curs_close()
    2064                          except PG_ERROR_EXCEPTION as pg_exc2:
    2065                                  _log.exception('cannot close cursor')
    2066                                  log_pg_exception_details(pg_exc2)
    2067                          try:
    2068                                  tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    2069                          except PG_ERROR_EXCEPTION as pg_exc2:
    2070                                  _log.exception('cannot rollback transaction')
    2071                                  log_pg_exception_details(pg_exc2)
    2072                          # privilege problem
    2073                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    2074                                  details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
    2075                                  if curs.statusmessage != '':
    2076                                          details = 'Status: %s\n%s' % (
    2077                                                  curs.statusmessage.strip().strip('\n').strip().strip('\n'),
    2078                                                  details
    2079                                          )
    2080                                  if pg_exc.pgerror is None:
    2081                                          msg = '[%s]' % pg_exc.pgcode
    2082                                  else:
    2083                                          msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
    2084                                  try:
    2085                                          curs_close()
    2086                                          tx_rollback()                   # just for good measure
    2087                                          conn_close()
    2088                                  except dbapi.InterfaceError:
    2089                                          _log.exception('cannot cleanup')
    2090                                  raise gmExceptions.AccessDenied (
    2091                                          msg,
    2092                                          source = 'PostgreSQL',
    2093                                          code = pg_exc.pgcode,
    2094                                          details = details
    2095                                  )
    2096                          # other problem
    2097                          gmLog2.log_stack_trace()
    2098                          try:
    2099                                  curs_close()
    2100                                  tx_rollback()                   # just for good measure
    2101                                  conn_close()
    2102                          except dbapi.InterfaceError:
    2103                                  _log.exception('cannot cleanup')
    2104                          raise
    2105                  # other exception
    2106                  except Exception:
    2107                          _log.exception('error running query in RW connection')
    2108                          _log.error(capture_cursor_state(curs))
    2109                          for notice in notices_accessor.notices:
    2110                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    2111                          del notices_accessor.notices[:]
    2112                          gmLog2.log_stack_trace()
    2113                          try:
    2114                                  curs_close()
    2115                          except PG_ERROR_EXCEPTION as pg_exc:
    2116                                  _log.exception('cannot close cursor')
    2117                                  log_pg_exception_details(pg_exc)
    2118                          try:
    2119                                  tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    2120                                  conn_close()
    2121                          except PG_ERROR_EXCEPTION as pg_exc:
    2122                                  _log.exception('cannot rollback transation')
    2123                                  log_pg_exception_details(pg_exc)
    2124                          raise
    2125
    2126          data = None
    2127          col_idx = None
    2128          if return_data:
    2129                  try:
    2130                          data = curs.fetchall()
    2131                  except Exception:
    2132                          _log.exception('error fetching data from RW query')
    2133                          gmLog2.log_stack_trace()
    2134                          try:
    2135                                  curs_close()
    2136                                  tx_rollback()
    2137                                  conn_close()
    2138                          except dbapi.InterfaceError:
    2139                                  _log.exception('cannot cleanup')
    2140                                  raise
    2141                          raise
    2142                  if get_col_idx:
    2143                          col_idx = get_col_indices(curs)
    2144
    2145          curs_close()
    2146          conn_commit()
    2147          conn_close()
    2148
    2149          return (data, col_idx)
    2150
    2151  #------------------------------------------------------------------------
    2152 -def run_insert(link_obj=None, schema=None, table=None, values=None, returning=None, end_tx=False, get_col_idx=False, verbose=False):
    2153          """Generates SQL for an INSERT query.
    2154
    2155          values: dict of values keyed by field to insert them into
    2156          """
    2157          if schema is None:
    2158                  schema = 'public'
    2159
    2160          fields = values.keys()          # that way val_snippets and fields really should end up in the same order
    2161          val_snippets = []
    2162          for field in fields:
    2163                  val_snippets.append('%%(%s)s' % field)
    2164
    2165          if returning is None:
    2166                  returning = ''
    2167                  return_data = False
    2168          else:
    2169                  returning = '\n\tRETURNING\n\t\t%s' % ', '.join(returning)
    2170                  return_data = True
    2171
    2172          cmd = """\nINSERT INTO %s.%s (
    2173                  %s
    2174          ) VALUES (
    2175                  %s
    2176          )%s""" % (
    2177                  schema,
    2178                  table,
    2179                  ',\n\t\t'.join(fields),
    2180                  ',\n\t\t'.join(val_snippets),
    2181                  returning
    2182          )
    2183
    2184          _log.debug('running SQL: >>>%s<<<', cmd)
    2185
    2186          return run_rw_queries (
    2187                  link_obj = link_obj,
    2188                  queries = [{'cmd': cmd, 'args': values}],
    2189                  end_tx = end_tx,
    2190                  return_data = return_data,
    2191                  get_col_idx = get_col_idx,
    2192                  verbose = verbose
    2193          )
    2194
    2195  # =======================================================================
    2196  # connection handling API
    2197  # -----------------------------------------------------------------------
    2198 -class cConnectionPool(psycopg2.pool.PersistentConnectionPool):
    2199          """GNUmed database connection pool.
    2200
    2201          Extends psycopg2's ThreadedConnectionPool with
    2202          a custom _connect() function. Supports one connection
    2203          per thread - which also ties it to one particular DSN."""
    2204          #--------------------------------------------------
    2205 -        def _connect(self, key=None):
    2206                  _log.debug('conn request with key [%s]', key)
    2207                  conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly = True)
    2208                  # monkey patching close()
    2209                  conn.original_close = conn.close
    2210                  conn.close = _raise_exception_on_ro_conn_close
    2211                  if key is not None:
    2212                          self._used[key] = conn
    2213                          self._rused[id(conn)] = key
    2214                  else:
    2215                          self._pool.append(conn)
    2216                  return conn
    2217
    2218          #--------------------------------------------------
    2219 -        def discard_connection(self, key=None):
    2220                  if key is None:
    2221                          key = threading.current_thread().ident
    2222                  try:
    2223                          conn = self._used[key]
    2224                  except KeyError:
    2225                          _log.error('no such key in connection pool: %s', key)
    2226                          _log.debug('available keys: %s', self._used.keys())
    2227                          return
    2228                  del self._used[key]
    2229                  del self._rused[id(conn)]
    2230                  conn.original_close()
    2231
    2232          #--------------------------------------------------
    2233 -        def shutdown(self):
    2234                  for conn_key in self._used.keys():
    2235                          conn = self._used[conn_key]
    2236                          if conn.closed != 0:
    2237                                  continue
    2238                          _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid())
    2239                          conn.original_close()
    2240
    2241  # -----------------------------------------------------------------------
    2242 -def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=None, autocommit=False):
    2243          """Get a raw, unadorned connection.
    2244
    2245          - this will not set any parameters such as encoding, timezone, datestyle
    2246          - the only requirement is a valid DSN
    2247          - hence it can be used for "service" connections
    2248            for verifying encodings etc
    2249          """
    2250          # FIXME: support verbose
    2251          if dsn is None:
    2252                  dsn = get_default_dsn()
    2253
    2254          if 'host=salaam.homeunix' in dsn:
    2255                  raise ValueError('The public database is not hosted by <salaam.homeunix.com> anymore.\n\nPlease point your configuration files to <publicdb.gnumed.de>.')
    2256
    2257          # try to enforce a useful encoding early on so that we
    2258          # have a good chance of decoding authentication errors
    2259          # containing foreign language characters
    2260          if ' client_encoding=' not in dsn:
    2261                  dsn += ' client_encoding=utf8'
    2262
    2263          if ' application_name' not in dsn:
    2264                  if connection_name is None:
    2265                          dsn += " application_name=GNUmed-[%s]"  % threading.current_thread().name.replace(' ', '_')
    2266                  else:
    2267                          dsn += " application_name=%s" % connection_name
    2268
    2269          try:
    2270                  # DictConnection now _is_ a real dictionary
    2271                  conn = dbapi.connect(dsn = dsn, connection_factory = psycopg2.extras.DictConnection)
    2272          except dbapi.OperationalError as e:
    2273                  t, v, tb = sys.exc_info()
    2274                  try:
    2275                          msg = e.args[0]
    2276                  except (AttributeError, IndexError, TypeError):
    2277                          raise
    2278                  if 'fe_sendauth' in msg:
    2279                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2280                  if regex.search('user ".*" does not exist', msg) is not None:
    2281                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2282                  if ((   (regex.search('user ".*"', msg) is not None)
    2283                                          or
    2284                                  (regex.search('(R|r)ol{1,2}e', msg) is not None)
    2285                          )
    2286                          and ('exist' in msg)
    2287                          and (regex.search('n(o|ich)t', msg) is not None)
    2288                  ):
    2289                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2290                  if regex.search('user ".*" does not exist', msg) is not None:
    2291                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2292                  if 'uthenti' in msg:
    2293                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2294                  raise
    2295
    2296          if connection_name is None:
    2297                  _log.debug('established anonymous database connection, backend PID: %s', conn.get_backend_pid())
    2298          else:
    2299                  _log.debug('established database connection "%s", backend PID: %s', connection_name, conn.get_backend_pid())
    2300
    2301          # do first-connection-only stuff
    2302          # - verify PG version
    2303          global postgresql_version
    2304          if postgresql_version is None:
    2305                  curs = conn.cursor()
    2306                  curs.execute("""
    2307                          SELECT
    2308                                  substring(setting, E'^\\\\d{1,2}\\\\.\\\\d{1,2}')::numeric AS version
    2309                          FROM
    2310                                  pg_settings
    2311                          WHERE
    2312                                  name = 'server_version'
    2313                  """)
    2314                  postgresql_version = curs.fetchone()['version']
    2315                  _log.info('PostgreSQL version (numeric): %s' % postgresql_version)
    2316                  try:
    2317                          curs.execute("SELECT pg_size_pretty(pg_database_size(current_database()))")
    2318                          _log.info('database size: %s', curs.fetchone()[0])
    2319                  except Exception:
    2320                          _log.exception('cannot get database size')
    2321                  finally:
    2322                          curs.close()
    2323                          conn.commit()
    2324                  if verbose:
    2325                          curs = conn.cursor()
    2326                          _log_PG_settings(curs = curs)
    2327                          curs.close()
    2328          # - verify PG understands client time zone
    2329          if _default_client_timezone is None:
    2330                  __detect_client_timezone(conn = conn)
    2331
    2332          # - set access mode
    2333          if readonly:
    2334                  _log.debug('readonly: forcing autocommit=True to avoid <IDLE IN TRANSACTION>')
    2335                  autocommit = True
    2336          else:
    2337                  _log.debug('autocommit is desired to be: %s', autocommit)
    2338
    2339          conn.commit()
    2340          conn.autocommit = autocommit
    2341          conn.readonly = readonly
    2342
    2343          # - assume verbose=True to mean we want debugging in the database, too
    2344          if verbose:
    2345                  _log.debug('enabling <plpgsql.extra_warnings/_errors>')
    2346                  curs = conn.cursor()
    2347                  try:
    2348                          curs.execute("SET plpgsql.extra_warnings TO 'all'")
    2349                          curs.execute("SET plpgsql.extra_errors TO 'all'")
    2350                  except Exception:
    2351                          _log.exception('cannot enable <plpgsql.extra_warnings/_errors>')
    2352                  finally:
    2353                          curs.close()
    2354                          conn.commit()
    2355
    2356          conn.is_decorated = False
    2357          return conn
    2358
    2359  # =======================================================================
    2360 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True, connection_name=None, autocommit=False):
    2361          """Get a new connection.
    2362
    2363          This assumes the locale system has been initialized
    2364          unless an encoding is specified.
    2365          """
    2366          # FIXME: support pooled on RW, too
    2367          # FIXME: for now, support the default DSN only
    2368          if pooled and readonly and (dsn is None):
    2369                  global __ro_conn_pool
    2370                  if __ro_conn_pool is None:
    2371                          log_ro_conn = True
    2372                          __ro_conn_pool = cConnectionPool (
    2373                                  minconn = 1,
    2374                                  maxconn = 2,
    2375                                  dsn = dsn,
    2376                                  verbose = verbose
    2377                          )
    2378                  else:
    2379                          log_ro_conn = False
    2380                  try:
    2381                          conn = __ro_conn_pool.getconn()
    2382                  except psycopg2.pool.PoolError:
    2383                          _log.exception('falling back to non-pooled connection')
    2384                          conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
    2385                          log_ro_conn = True
    2386                  if log_ro_conn:
    2387                          [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
    2388          else:
    2389                  conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
    2390
    2391          if conn.is_decorated:
    2392                  return conn
    2393
    2394          if encoding is None:
    2395                  encoding = _default_client_encoding
    2396          if encoding is None:
    2397                  encoding = gmI18N.get_encoding()
    2398                  _log.warning('client encoding not specified')
    2399                  _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding)
    2400                  _log.warning('for this to work properly the application MUST have called locale.setlocale() before')
    2401
    2402          # set connection properties
    2403          # - client encoding
    2404          try:
    2405                  conn.set_client_encoding(encoding)
    2406          except dbapi.DataError:
    2407                  t, v, tb = sys.exc_info()
    2408                  if 'cannot set encoding to' in str(v):
    2409                          raise cEncodingError(encoding, v).with_traceback(tb)
    2410                  if 'invalid value for parameter "client_encoding"' in str(v):
    2411                          raise cEncodingError(encoding, v).with_traceback(tb)
    2412                  raise
    2413
    2414          # - transaction isolation level
    2415          if readonly:
    2416                  # alter-database default, checked at connect, no need to set here
    2417                  pass
    2418          else:
    2419                  conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE)
    2420
    2421          _log.debug('client time zone [%s]', _default_client_timezone)
    2422
    2423          # - client time zone
    2424          curs = conn.cursor()
    2425          curs.execute(_sql_set_timezone, [_default_client_timezone])
    2426          curs.close()
    2427          conn.commit()
    2428
    2429          conn.is_decorated = True
    2430
    2431          if verbose:
    2432                  [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
    2433
    2434          return conn
    2435
    2436  #-----------------------------------------------------------------------
    2437 -def discard_pooled_connection(conn_key=None):
    2438          if __ro_conn_pool is None:
    2439                  return
    2440          __ro_conn_pool.discard_connection(key = conn_key)
    2441
    2442  #-----------------------------------------------------------------------
    2443 -def shutdown():
    2444          if __ro_conn_pool is None:
    2445                  return
    2446          __ro_conn_pool.shutdown()
    2447
    2448  # ======================================================================
    2449  # internal helpers
    2450  #-----------------------------------------------------------------------
    2451 -def __noop():
    2452          pass
    2453
    2454  #-----------------------------------------------------------------------
    2455 -def _raise_exception_on_ro_conn_close():
    2456          raise TypeError('close() called on read-only connection')
    2457
    2458  #-----------------------------------------------------------------------
    2459 -def log_database_access(action=None):
    2460          run_insert (
    2461                  schema = 'gm',
    2462                  table = 'access_log',
    2463                  values = {'user_action': action},
    2464                  end_tx = True
    2465          )
    2466
    2467  #-----------------------------------------------------------------------
    2468 -def sanity_check_time_skew(tolerance=60):
    2469          """Check server time and local time to be within
    2470          the given tolerance of each other.
    2471
    2472          tolerance: seconds
    2473          """
    2474          _log.debug('maximum skew tolerance (seconds): %s', tolerance)
    2475
    2476          cmd = "SELECT now() at time zone 'UTC'"
    2477          conn = get_raw_connection(readonly=True)
    2478          curs = conn.cursor()
    2479
    2480          start = time.time()
    2481          rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}])
    2482          end = time.time()
    2483          client_now_as_utc = pydt.datetime.utcnow()
    2484
    2485          curs.close()
    2486          conn.commit()
    2487
    2488          server_now_as_utc = rows[0][0]
    2489          query_duration = end - start
    2490          _log.info('server "now" (UTC): %s', server_now_as_utc)
    2491          _log.info('client "now" (UTC): %s', client_now_as_utc)
    2492          _log.debug('wire roundtrip (seconds): %s', query_duration)
    2493
    2494          if query_duration > tolerance:
    2495                  _log.error('useless to check client/server time skew, wire roundtrip > tolerance')
    2496                  return False
    2497
    2498          if server_now_as_utc > client_now_as_utc:
    2499                  real_skew = server_now_as_utc - client_now_as_utc
    2500          else:
    2501                  real_skew = client_now_as_utc - server_now_as_utc
    2502
    2503          _log.debug('client/server time skew: %s', real_skew)
    2504
    2505          if real_skew > pydt.timedelta(seconds = tolerance):
    2506                  _log.error('client/server time skew > tolerance')
    2507                  return False
    2508
    2509          return True
    2510
    2511  #-----------------------------------------------------------------------
    2512 -def sanity_check_database_settings():
    2513          """Checks database settings.
    2514
    2515          returns (status, message)
    2516          status:
    2517                  0: no problem
    2518                  1: non-fatal problem
    2519                  2: fatal problem
    2520          """
    2521          _log.debug('checking database settings')
    2522
    2523          conn = get_connection()
    2524
    2525          # - version string
    2526          global postgresql_version_string
    2527          if postgresql_version_string is None:
    2528                  curs = conn.cursor()
    2529                  curs.execute('SELECT version()')
    2530                  postgresql_version_string = curs.fetchone()['version']
    2531                  curs.close()
    2532                  _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string)
    2533
    2534          options2check = {
    2535                  # setting: [expected value, risk, fatal?]
    2536                  'allow_system_table_mods': [['off'], 'system breakage', False],
    2537                  'check_function_bodies': [['on'], 'suboptimal error detection', False],
    2538                  'datestyle': [['ISO'], 'faulty timestamp parsing', True],
    2539                  'default_transaction_isolation': [['read committed'], 'faulty database reads', True],
    2540                  'default_transaction_read_only': [['on'], 'accidental database writes', False],
    2541                  'fsync': [['on'], 'data loss/corruption', True],
    2542                  'full_page_writes': [['on'], 'data loss/corruption', False],
    2543                  'lc_messages': [['C'], 'suboptimal error detection', False],
    2544                  'password_encryption': [['on', 'md5', 'scram-sha-256'], 'breach of confidentiality', False],
    2545                  #u'regex_flavor': [[u'advanced'], u'query breakage', False],                                    # 9.0 doesn't support this anymore, default now advanced anyway
    2546                  'synchronous_commit': [['on'], 'data loss/corruption', False],
    2547                  'sql_inheritance': [['on'], 'query breakage, data loss/corruption', True],      # IF returned (<PG10): better be ON, if NOT returned (PG10): hardwired
    2548                  'ignore_checksum_failure': [['off'], 'data loss/corruption', False],            # starting with PG 9.3
    2549                  'track_commit_timestamp': [['on'], 'suboptimal auditing', False]                        # starting with PG 9.3
    2550          }
    2551
    2552          from Gnumed.pycommon import gmCfg2
    2553          _cfg = gmCfg2.gmCfgData()
    2554          if _cfg.get(option = 'hipaa'):
    2555                  options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', True]
    2556                  options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', True]
    2557          else:
    2558                  options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', None]
    2559                  options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', None]
    2560
    2561          cmd = "SELECT name, setting from pg_settings where name in %(settings)s"
    2562          rows, idx = run_ro_queries (
    2563                  link_obj = conn,
    2564                  queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}],
    2565                  get_col_idx = False
    2566          )
    2567
    2568          found_error = False
    2569          found_problem = False
    2570          msg = []
    2571          for row in rows:
    2572                  option = row['name']
    2573                  value_found = row['setting']
    2574                  values_expected = options2check[option][0]
    2575                  risk = options2check[option][1]
    2576                  fatal_setting = options2check[option][2]
    2577                  if value_found not in values_expected:
    2578                          if fatal_setting is True:
    2579                                  found_error = True
    2580                          elif fatal_setting is False:
    2581                                  found_problem = True
    2582                          elif fatal_setting is None:
    2583                                  pass
    2584                          else:
    2585                                  _log.error(options2check[option])
    2586                                  raise ValueError('invalid database configuration sanity check')
    2587                          msg.append(_(' option [%s]: %s') % (option, value_found))
    2588                          msg.append(_('  risk: %s') % risk)
    2589                          _log.warning('PG option [%s] set to [%s], expected %s, risk: <%s>' % (option, value_found, values_expected, risk))
    2590
    2591          if found_error:
    2592                  return 2, '\n'.join(msg)
    2593
    2594          if found_problem:
    2595                  return 1, '\n'.join(msg)
    2596
    2597          return 0, ''
    2598
    2599  #------------------------------------------------------------------------
    2600 -def _log_PG_settings(curs=None):
    2601          # don't use any of the run_*()s helper functions
    2602          # since that might create a loop if we fail here
    2603          try:
    2604                  curs.execute('SELECT * FROM pg_settings')
    2605          except dbapi.Error:
    2606                  _log.exception('cannot retrieve PG settings ("SELECT ... FROM pg_settings" failed)')
    2607                  return False
    2608          settings = curs.fetchall()
    2609          for setting in settings:
    2610                  if setting['unit'] is None:
    2611                          unit = ''
    2612                  else:
    2613                          unit = ' %s' % setting['unit']
    2614                  if setting['sourcefile'] is None:
    2615                          sfile = ''
    2616                  else:
    2617                          sfile = '// %s @ %s' % (setting['sourcefile'], setting['sourceline'])
    2618                  pending_restart = u''
    2619                  try:
    2620                          if setting['pending_restart']:
    2621                                  pending_restart = u'// needs restart'
    2622                  except KeyError:
    2623                          # 'pending_restart' does not exist in PG 9.4 yet
    2624                          pass
    2625                  _log.debug('%s: %s%s (set from: [%s] // sess RESET will set to: [%s]%s%s)',
    2626                          setting['name'],
    2627                          setting['setting'],
    2628                          unit,
    2629                          setting['source'],
    2630                          setting['reset_val'],
    2631                          pending_restart,
    2632                          sfile
    2633                  )
    2634          try:
    2635                  curs.execute('select pg_available_extensions()')
    2636          except:
    2637                  _log.exception('cannot log available PG extensions')
    2638                  return False
    2639          extensions = curs.fetchall()
    2640          if extensions is None:
    2641                  _log.error('no PG extensions available')
    2642                  return False
    2643          for ext in extensions:
    2644                  _log.debug('PG extension: %s', ext['pg_available_extensions'])
    2645
    2646          # not really that useful because:
    2647          # - clusterwide
    2648          # - not retained across server restart (fixed in 9.6.1 - really ?)
    2649  #       try:
    2650  #               curs.execute(u'SELECT pg_last_committed_xact()')
    2651  #       except:
    2652  #               _log.exception(u'cannot retrieve last committed xact')
    2653  #       xact = curs.fetchall()
    2654  #       if xact is not None:
    2655  #               _log.debug(u'last committed transaction in cluster: %s', xact[0])
    2656
    2657          return True
    2658
    2659  #------------------------------------------------------------------------
    2660 -def log_pg_exception_details(exc):
    2661          if not isinstance(exc, dbapi.Error):
    2662                  return False
    2663          try:
    2664                  args = exc.args
    2665                  for arg in args:
    2666                          _log.debug('exc.arg: %s', arg)
    2667          except AttributeError:
    2668                  _log.debug('exception has no <.args>')
    2669          _log.debug('pgerror: [%s]', exc.pgerror)
    2670          if exc.pgcode is None:
    2671                  _log.debug('pgcode : %s', exc.pgcode)
    2672          else:
    2673                  _log.debug('pgcode : %s (%s)', exc.pgcode, sql_error_codes.lookup(exc.pgcode))
    2674          if exc.cursor is None:
    2675                  _log.debug('cursor: None')
    2676          else:
    2677                  capture_cursor_state(cursor = exc.cursor)
    2678          try:
    2679                  exc.diag
    2680                  for attr in dir(exc.diag):
    2681                          if attr.startswith('__'):
    2682                                  continue
    2683                          val = getattr(exc.diag, attr)
    2684                          if val is None:
    2685                                  continue
    2686                          _log.debug('%s: %s', attr, val)
    2687          except AttributeError:
    2688                  _log.debug('diag: not available')
    2689          return True
    2690
    2691  #------------------------------------------------------------------------
    2692 -def exception_is_connection_loss(exc):
    2693          if not isinstance(exc, dbapi.Error):
    2694                  # not a PG exception
    2695                  return False
    2696          try:
    2697                  msg = '%s' % exc.args[0]
    2698          except (AttributeError, IndexError, TypeError):
    2699                  _log.debug('cannot extract message from exception')
    2700                  # cannot process message
    2701                  return False
    2702          _log.debug('interpreting: %s', msg)
    2703          # OperationalError
    2704          conn_lost = (
    2705                  ('erver' in msg)
    2706                          and
    2707                  (
    2708                          ('terminat' in msg)
    2709                                  or
    2710                          ('abnorm' in msg)
    2711                                  or
    2712                          ('end' in msg)
    2713  #                               or
    2714  #                       ('oute' in msg)
    2715                  )
    2716          )
    2717          if conn_lost:
    2718                  _log.debug('indicates connection loss')
    2719                  return True
    2720          # InterfaceError
    2721          conn_lost = (
    2722                  ('onnect' in msg)
    2723                          and
    2724                  (
    2725                          ('close' in msg)
    2726                                  or
    2727                          ('end' in msg)
    2728                  )
    2729          )
    2730          if conn_lost:
    2731                  _log.debug('indicates connection loss')
    2732          return conn_lost
    2733
    2734  #========================================================================
    2735 -class cAuthenticationError(dbapi.OperationalError):
    2736
    2737 -        def __init__(self, dsn=None, prev_val=None):
    2738                  self.dsn = dsn
    2739                  self.prev_val = prev_val
    2740
    2741 -        def __str__(self):
    2742                  return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
    2743
    2744  #========================================================================
    2745  # custom psycopg2 extensions
    2746  #========================================================================
    2747 -class cEncodingError(dbapi.OperationalError):
    2748
    2749 -        def __init__(self, encoding=None, prev_val=None):
    2750                  self.encoding = encoding
    2751                  self.prev_val = prev_val
    2752
    2753 -        def __str__(self):
    2754                  return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
    2755
    2756  # -----------------------------------------------------------------------
    2757  # Python -> PostgreSQL
    2758  # -----------------------------------------------------------------------
    2759  # test when Squeeze (and thus psycopg2 2.2 becomes Stable
    2760 -class cAdapterPyDateTime(object):
    2761
    2762 -        def __init__(self, dt):
    2763                  if dt.tzinfo is None:
    2764                          raise ValueError('datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat())
    2765                  self.__dt = dt
    2766
    2767 -        def getquoted(self):
    2768                  return _timestamp_template % self.__dt.isoformat()
    2769
    2770  #=======================================================================
    2771  #  main
    2772  #-----------------------------------------------------------------------
    2773
    2774  # make sure psycopg2 knows how to handle unicode ...
    2775  psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
    2776  psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY)
    2777
    2778  # tell psycopg2 how to adapt datetime types with timestamps when locales are in use
    2779  # check in 0.9:
    2780  psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime)
    2781
    2782  # turn dict()s into JSON - only works > 9.2
    2783  #psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
    2784
    2785  # do NOT adapt *lists* to "... IN (*) ..." syntax because we want
    2786  # them adapted to "... ARRAY[]..." so we can support PG arrays
    2787
    2788  #=======================================================================
    2789  if __name__ == "__main__":
    2790
    2791          if len(sys.argv) < 2:
    2792                  sys.exit()
    2793
    2794          if sys.argv[1] != 'test':
    2795                  sys.exit()
    2796
    2797          from Gnumed.pycommon.gmTools import file2md5
    2798
    2799          logging.basicConfig(level=logging.DEBUG)
    2800
    2801          #--------------------------------------------------------------------
    2802 -        def test_file2bytea():
    2803                  run_rw_queries(queries = [
    2804                          {'cmd': 'drop table if exists test_bytea'},
    2805                          {'cmd': 'create table test_bytea (data bytea)'}
    2806                  ])
    2807
    2808                  try:
    2809                          file2bytea(query = 'insert into test_bytea values (%(data)s::bytea) returning md5(data) as md5', filename = sys.argv[2], file_md5 = file2md5(sys.argv[2], True))
    2810                  except:
    2811                          _log.exception('error')
    2812
    2813                  run_rw_queries(queries = [
    2814                          {'cmd': 'drop table test_bytea'}
    2815                  ])
    2816
    2817          #--------------------------------------------------------------------
    2818 -        def test_file2bytea_lo():
    2819                  lo_oid = file2bytea_lo (
    2820                          filename = sys.argv[2]
    2821                          #, file_md5 = file2md5(sys.argv[2], True)
    2822                  )
    2823                  print(lo_oid)
    2824  #               if lo_oid != -1:
    2825  #                       run_rw_queries(queries = [
    2826  #                               {'cmd': u'select lo_unlink(%(loid)s::oid)', 'args': {'loid': lo_oid}}
    2827  #                       ])
    2828
    2829          #--------------------------------------------------------------------
    2830 -        def test_file2bytea_copy_from():
    2831
    2832                  run_rw_queries(queries = [
    2833                          {'cmd': 'drop table if exists test_bytea'},
    2834                          {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
    2835                          {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
    2836                  ])
    2837
    2838                  md5_query = {
    2839                          'cmd': 'select md5(data) AS md5 FROM test_bytea WHERE pk = %(pk)s',
    2840                          'args': {'pk': 1}
    2841                  }
    2842
    2843                  file2bytea_copy_from (
    2844                          table = 'test_bytea',
    2845                          columns = ['data'],
    2846                          filename = sys.argv[2],
    2847                          md5_query = md5_query,
    2848                          file_md5 = file2md5(sys.argv[2], True)
    2849                  )
    2850
    2851                  run_rw_queries(queries = [
    2852                          {'cmd': 'drop table if exists test_bytea'}
    2853                  ])
    2854
    2855          #--------------------------------------------------------------------
    2856 -        def test_file2bytea_overlay():
    2857
    2858                  run_rw_queries(queries = [
    2859                          {'cmd': 'drop table if exists test_bytea'},
    2860                          {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
    2861                          {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
    2862                  ])
    2863
    2864                  cmd = """
    2865                  update test_bytea
    2866                  set data = overlay (
    2867                          coalesce(data, ''::bytea)
    2868                          placing %(data)s::bytea
    2869                          from %(start)s
    2870                          for %(size)s
    2871                  )
    2872                  where
    2873                          pk > %(pk)s
    2874                  """
    2875                  md5_cmd = 'select md5(data) from test_bytea'
    2876                  args = {'pk': 0}
    2877                  file2bytea_overlay (
    2878                          query = cmd,
    2879                          args = args,
    2880                          filename = sys.argv[2],
    2881                          conn = None,
    2882                          md5_query = md5_cmd,
    2883                          file_md5 = file2md5(sys.argv[2], True)
    2884                  )
    2885
    2886                  run_rw_queries(queries = [
    2887                          {'cmd': 'drop table test_bytea'}
    2888                  ])
    2889
    2890          #--------------------------------------------------------------------
    2891 -        def test_get_connection():
    2892                  print("testing get_connection()")
    2893
    2894                  print('')
    2895                  dsn = 'foo'
    2896                  try:
    2897                          conn = get_connection(dsn=dsn)
    2898                  except dbapi.ProgrammingError as e:
    2899                          print("1) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2900                          t, v = sys.exc_info()[:2]
    2901                          print (' ', t)
    2902                          print (' ', v)
    2903
    2904                  print('')
    2905                  dsn = 'dbname=gnumed_v9'
    2906                  try:
    2907                          conn = get_connection(dsn=dsn)
    2908                  except cAuthenticationError:
    2909                          print("2) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2910                          t, v = sys.exc_info()[:2]
    2911                          print(' ', t)
    2912                          print(' ', v)
    2913
    2914                  print('')
    2915                  dsn = 'dbname=gnumed_v9 user=abc'
    2916                  try:
    2917                          conn = get_connection(dsn=dsn)
    2918                  except cAuthenticationError:
    2919                          print("3) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2920                          t, v = sys.exc_info()[:2]
    2921                          print(' ', t)
    2922                          print(' ', v)
    2923
    2924                  print('')
    2925                  dsn = 'dbname=gnumed_v9 user=any-doc password=abc'
    2926                  try:
    2927                          conn = get_connection(dsn=dsn)
    2928                  except cAuthenticationError:
    2929                          print("5) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2930                          t, v = sys.exc_info()[:2]
    2931                          print(' ', t)
    2932                          print(' ', v)
    2933
    2934                  print('')
    2935                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2936                  conn = get_connection(dsn=dsn, readonly=True)
    2937
    2938                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2939                  conn = get_connection(dsn=dsn, readonly=False, verbose=True)
    2940
    2941                  print('')
    2942                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2943                  encoding = 'foo'
    2944                  try:
    2945                          conn = get_connection(dsn=dsn, encoding=encoding)
    2946                  except cEncodingError:
    2947                          print("6) SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding))
    2948                          t, v = sys.exc_info()[:2]
    2949                          print(' ', t)
    2950                          print(' ', v)
    2951
    2952                  print('')
    2953                  dsn = 'dbname=gnumed_v22 user=any-doc'
    2954                  try:
    2955                          conn = get_connection(dsn=dsn)
    2956                          print("6) SUCCESS:", dsn)
    2957                          print('pid:', conn.get_backend_pid())
    2958                  except cAuthenticationError:
    2959                          print("4) SUCCESS: get_connection(%s) failed" % dsn)
    2960                          t, v = sys.exc_info()[:2]
    2961                          print(' ', t)
    2962                          print(' ', v)
    2963
    2964                  try:
    2965                          curs = conn.cursor()
    2966                          input('hit enter to run query')
    2967                          curs.execute('selec 1')
    2968                  except Exception as exc:
    2969                          print('ERROR')
    2970                          _log.exception('exception occurred')
    2971                          log_pg_exception_details(exc)
    2972                          if exception_is_connection_loss(exc):
    2973                                  _log.error('lost connection')
    2974
    2975          #--------------------------------------------------------------------
    2976 -        def test_exceptions():
    2977                  print("testing exceptions")
    2978
    2979                  try:
    2980                          raise cAuthenticationError('no dsn', 'no previous exception')
    2981                  except cAuthenticationError:
    2982                          t, v, tb = sys.exc_info()
    2983                          print(t)
    2984                          print(v)
    2985                          print(tb)
    2986
    2987                  try:
    2988                          raise cEncodingError('no dsn', 'no previous exception')
    2989                  except cEncodingError:
    2990                          t, v, tb = sys.exc_info()
    2991                          print(t)
    2992                          print(v)
    2993                          print(tb)
    2994          #--------------------------------------------------------------------
    2995 -        def test_ro_queries():
    2996                  print("testing run_ro_queries()")
    2997
    2998                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2999                  conn = get_connection(dsn, readonly=True)
    3000
    3001                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    3002                  print(data)
    3003                  print(idx)
    3004                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True)
    3005                  print(data)
    3006                  print(idx)
    3007
    3008                  curs = conn.cursor()
    3009
    3010                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    3011                  print(data)
    3012                  print(idx)
    3013
    3014                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True, verbose=True)
    3015                  print(data)
    3016                  print(idx)
    3017
    3018                  try:
    3019                          data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'selec 1'}], return_data=True, get_col_idx=True, verbose=True)
    3020                          print(data)
    3021                          print(idx)
    3022                  except psycopg2.ProgrammingError:
    3023                          print('SUCCESS: run_ro_queries("selec 1") failed as expected')
    3024                          t, v = sys.exc_info()[:2]
    3025                          print(' ', t)
    3026                          print(' ', v)
    3027
    3028                  curs.close()
    3029
    3030          #--------------------------------------------------------------------
    3031 -        def test_request_dsn():
    3032                  conn = get_connection()
    3033                  print(conn)
    3034                  conn.close()
    3035          #--------------------------------------------------------------------
    3036 -        def test_set_encoding():
    3037                  print("testing set_default_client_encoding()")
    3038
    3039                  enc = 'foo'
    3040                  try:
    3041                          set_default_client_encoding(enc)
    3042                          print("SUCCESS: encoding [%s] worked" % enc)
    3043                  except ValueError:
    3044                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3045                          t, v = sys.exc_info()[:2]
    3046                          print(' ', t)
    3047                          print(' ', v)
    3048
    3049                  enc = ''
    3050                  try:
    3051                          set_default_client_encoding(enc)
    3052                          print("SUCCESS: encoding [%s] worked" % enc)
    3053                  except ValueError:
    3054                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3055                          t, v = sys.exc_info()[:2]
    3056                          print(' ', t)
    3057                          print(' ', v)
    3058
    3059                  enc = 'latin1'
    3060                  try:
    3061                          set_default_client_encoding(enc)
    3062                          print("SUCCESS: encoding [%s] worked" % enc)
    3063                  except ValueError:
    3064                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3065                          t, v = sys.exc_info()[:2]
    3066                          print(' ', t)
    3067                          print(' ', v)
    3068
    3069                  enc = 'utf8'
    3070                  try:
    3071                          set_default_client_encoding(enc)
    3072                          print("SUCCESS: encoding [%s] worked" % enc)
    3073                  except ValueError:
    3074                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3075                          t, v = sys.exc_info()[:2]
    3076                          print(' ', t)
    3077                          print(' ', v)
    3078
    3079                  enc = 'unicode'
    3080                  try:
    3081                          set_default_client_encoding(enc)
    3082                          print("SUCCESS: encoding [%s] worked" % enc)
    3083                  except ValueError:
    3084                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3085                          t, v = sys.exc_info()[:2]
    3086                          print(' ', t)
    3087                          print(' ', v)
    3088
    3089                  enc = 'UNICODE'
    3090                  try:
    3091                          set_default_client_encoding(enc)
    3092                          print("SUCCESS: encoding [%s] worked" % enc)
    3093                  except ValueError:
    3094                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    3095                          t, v = sys.exc_info()[:2]
    3096                          print(' ', t)
    3097                          print(' ', v)
    3098          #--------------------------------------------------------------------
    3099 -        def test_connection_pool():
    3100                  dsn = get_default_dsn()
    3101                  pool = cConnectionPool(minconn=1, maxconn=2, dsn=None, verbose=False)
    3102                  print(pool)
    3103                  print(pool.getconn())
    3104                  print(pool.getconn())
    3105                  print(pool.getconn())
    3106                  print(type(pool.getconn()))
    3107          #--------------------------------------------------------------------
    3108 -        def test_list_args():
    3109                  dsn = get_default_dsn()
    3110                  conn = get_connection(dsn, readonly=True)
    3111                  curs = conn.cursor()
    3112                  curs.execute('SELECT * from clin.clin_narrative where narrative = %s', ['a'])
    3113          #--------------------------------------------------------------------
    3114 -        def test_sanitize_pg_regex():
    3115                  tests = [
    3116                          ['(', '\\(']
    3117                          , ['[', '\\[']
    3118                          , [')', '\\)']
    3119                  ]
    3120                  for test in tests:
    3121                          result = sanitize_pg_regex(test[0])
    3122                          if result != test[1]:
    3123                                  print('ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
    3124          #--------------------------------------------------------------------
    3125 -        def test_is_pg_interval():
    3126                  status = True
    3127                  tests = [
    3128                          [None, True],           # None == NULL == succeeds !
    3129                          [1, True],
    3130                          ['1', True],
    3131                          ['abc', False]
    3132                  ]
    3133
    3134                  if not is_pg_interval():
    3135                          print('ERROR: is_pg_interval() returned "False", expected "True"')
    3136                          status = False
    3137
    3138                  for test in tests:
    3139                          result = is_pg_interval(test[0])
    3140                          if result != test[1]:
    3141                                  print('ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
    3142                                  status = False
    3143
    3144                  return status
    3145          #--------------------------------------------------------------------
    3146 -        def test_sanity_check_time_skew():
    3147                  sanity_check_time_skew()
    3148
    3149          #--------------------------------------------------------------------
    3150 -        def test_get_foreign_key_names():
    3151                  print(get_foreign_key_names (
    3152                          src_schema = 'clin',
    3153                          src_table = 'vaccination',
    3154                          src_column = 'fk_episode',
    3155                          target_schema = 'clin',
    3156                          target_table = 'episode',
    3157                          target_column = 'pk'
    3158                  ))
    3159
    3160          #--------------------------------------------------------------------
    3161 -        def test_get_foreign_key_details():
    3162                  schema = 'clin'
    3163                  table = 'episode'
    3164                  col = 'pk'
    3165                  print('column %s.%s.%s is referenced by:' % (schema, table, col))
    3166                  for row in get_foreign_keys2column (
    3167                          schema = schema,
    3168                          table = table,
    3169                          column = col
    3170                  ):
    3171                          print(' <- %s.%s' % (
    3172                                  row['referencing_table'],
    3173                                  row['referencing_column']
    3174                          ))
    3175
    3176          #--------------------------------------------------------------------
    3177 -        def test_set_user_language():
    3178                  # (user, language, result, exception type)
    3179                  tests = [
    3180                          # current user
    3181                          [None, 'de_DE', True],
    3182                          [None, 'lang_w/o_tx', False],
    3183                          [None, None, True],
    3184                          # valid user
    3185                          ['any-doc', 'de_DE', True],
    3186                          ['any-doc', 'lang_w/o_tx', False],
    3187                          ['any-doc', None, True],
    3188                          # invalid user
    3189                          ['invalid user', 'de_DE', None],
    3190                          ['invalid user', 'lang_w/o_tx', False], # lang checking happens before user checking
    3191                          ['invalid user', None, True]
    3192                  ]
    3193                  for test in tests:
    3194                          try:
    3195                                  result = set_user_language(user = test[0], language = test[1])
    3196                                  if result != test[2]:
    3197                                          print("test:", test)
    3198                                          print("result:", result, "expected:", test[2])
    3199                          except psycopg2.IntegrityError as e:
    3200                                  if test[2] is None:
    3201                                          continue
    3202                                  print("test:", test)
    3203                                  print("expected exception")
    3204                                  print("result:", e)
    3205
    3206          #--------------------------------------------------------------------
    3207 -        def test_get_schema_revision_history():
    3208                  for line in get_schema_revision_history():
    3209                          print(' - '.join(line))
    3210
    3211          #--------------------------------------------------------------------
    3212 -        def test_run_query():
    3213                  gmDateTime.init()
    3214                  args = {'dt': gmDateTime.pydt_max_here()}
    3215                  cmd = "SELECT %(dt)s"
    3216
    3217                  #cmd = u"SELECT 'infinity'::timestamp with time zone"
    3218
    3219                  cmd = """
    3220  SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
    3221          SELECT REGEXP_REPLACE (
    3222                  't1.130729.0902.tif',                   -- string
    3223                  E'(.1)\.([0-9\.]+)(\.tif)',             -- pattern
    3224                  E'\\\\2'                                                -- replacement
    3225          ) AS foofoo
    3226  ) AS foo"""
    3227                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
    3228                  print(rows)
    3229                  print(rows[0])
    3230                  print(rows[0][0])
    3231          #--------------------------------------------------------------------
    3232 -        def test_schema_exists():
    3233                  print(schema_exists())
    3234          #--------------------------------------------------------------------
    3235 -        def test_row_locks():
    3236                  row_is_locked(table = 'dem.identity', pk = 12)
    3237
    3238                  print("1st connection:")
    3239                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3240                  print(" 1st shared lock succeeded:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3241                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3242
    3243                  print("   2nd shared lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3244                  print("   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3245                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3246                  print("   exclusive lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = True))
    3247                  print("   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = True))
    3248                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3249
    3250                  print("2nd connection:")
    3251                  conn = get_raw_connection(readonly=True)
    3252                  print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3253                  print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3254                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3255                  print(" exclusive lock succeeded ?", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True), "(should fail)")
    3256                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3257
    3258                  print("1st connection:")
    3259                  print(" unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3260                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3261
    3262                  print("2nd connection:")
    3263                  print(" exclusive lock should succeed", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True))
    3264                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3265                  print("  shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3266                  print("  `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3267                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3268                  print(" unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3269                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3270
    3271                  conn.close()
    3272
    3273          #--------------------------------------------------------------------
    3274 -        def test_get_foreign_key_names():
    3275                  print(get_foreign_key_names (
    3276                          src_schema = 'dem',
    3277                          src_table = 'names',
    3278                          src_column = 'id_identity',
    3279                          target_schema = 'dem',
    3280                          target_table = 'identity',
    3281                          target_column = 'pk'
    3282                  ))
    3283
    3284          #--------------------------------------------------------------------
    3285 -        def test_get_index_name():
    3286                  print(get_index_name(indexed_table = 'clin.vaccination', indexed_column = 'fk_episode'))
    3287
    3288          #--------------------------------------------------------------------
    3289 -        def test_faulty_SQL():
    3290                  run_rw_queries(queries = [{'cmd': 'SELEC 1'}])
    3291
    3292          #--------------------------------------------------------------------
    3293 -        def test_log_settings():
    3294                  conn = conn = get_connection()
    3295                  _log_PG_settings(curs = conn.cursor())
    3296
    3297          #--------------------------------------------------------------------
    3298          # run tests
    3299          #test_get_connection()
    3300          #test_exceptions()
    3301          test_ro_queries()
    3302          #test_request_dsn()
    3303          #test_set_encoding()
    3304          #test_connection_pool()
    3305          #test_list_args()
    3306          #test_sanitize_pg_regex()
    3307          #test_is_pg_interval()
    3308          #test_sanity_check_time_skew()
    3309          #test_get_foreign_key_details()
    3310          #test_get_foreign_key_names()
    3311          #test_get_index_name()
    3312          #test_set_user_language()
    3313          #test_get_schema_revision_history()
    3314          #test_run_query()
    3315          #test_schema_exists()
    3316          #test_get_foreign_key_names()
    3317          #test_row_locks()
    3318          #test_file2bytea()
    3319          #test_file2bytea_overlay()
    3320          #test_file2bytea_copy_from()
    3321          #test_file2bytea_lo()
    3322          #test_faulty_SQL()
    3323          #test_log_settings()
    3324
    3325  # ======================================================================
    3326