Package Gnumed :: Package pycommon :: Module gmBackendListener |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmBackendListener

      1  __doc__ = """GNUmed database backend listener.
      2
      3  This module implements threaded listening for asynchronuous
      4  notifications from the database backend.
      5  """
      6  #=====================================================================
      7  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>"
      8  __license__ = "GPL v2 or later"
      9
     10  import sys
     11  import time
     12  import threading
     13  import select
     14  import logging
     15
     16
     17  if __name__ == '__main__':
     18          sys.path.insert(0, '../../')
     19  from Gnumed.pycommon import gmDispatcher
     20  from Gnumed.pycommon import gmBorg
     21
     22
     23  _log = logging.getLogger('gm.db')
     24
     25
     26  signals2listen4 = [
     27          'db_maintenance_warning',               # warns of impending maintenance and asks for disconnect
     28          'db_maintenance_disconnect',    # announces a forced disconnect and disconnects
     29          'gm_table_mod'                                  # sent for any (registered) table modification, payload contains details
     30  ]
     31
     32  #=====================================================================
     33 -class gmBackendListener(gmBorg.cBorg):
     34
     35 -        def __init__(self, conn=None, poll_interval=3):
     36
     37                  try:
     38                          self.already_inited
     39                          return
     40                  except AttributeError:
     41                          pass
     42
     43                  self.debug = False
     44                  self.__notifications_received = 0
     45                  self.__messages_sent = 0
     46
     47                  _log.info('starting backend notifications listener thread')
     48
     49                  # the listener thread will regularly try to acquire
     50                  # this lock, when it succeeds it will quit
     51                  self._quit_lock = threading.Lock()
     52                  # take the lock now so it cannot be taken by the worker
     53                  # thread until it is released in shutdown()
     54                  if not self._quit_lock.acquire(0):
     55                          _log.error('cannot acquire thread-quit lock, aborting')
     56                          raise EnvironmentError("cannot acquire thread-quit lock")
     57
     58                  self._conn = conn
     59                  self.backend_pid = self._conn.get_backend_pid()
     60                  _log.debug('notification listener connection has backend PID [%s]', self.backend_pid)
     61                  self._conn.set_isolation_level(0)               # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT
     62                  self._cursor = self._conn.cursor()
     63                  try:
     64                          self._conn_fd = self._conn.fileno()
     65                  except AttributeError:
     66                          self._conn_fd = self._cursor.fileno()
     67                  self._conn_lock = threading.Lock()              # lock for access to connection object
     68
     69                  self.__register_interests()
     70
     71                  # check for messages every 'poll_interval' seconds
     72                  self._poll_interval = poll_interval
     73                  self._listener_thread = None
     74                  self.__start_thread()
     75
     76                  self.already_inited = True
     77
     78          #-------------------------------
     79          # public API
     80          #-------------------------------
     81 -        def shutdown(self):
     82                  _log.debug('received %s notifications', self.__notifications_received)
     83                  _log.debug('sent %s messages', self.__messages_sent)
     84
     85                  if self._listener_thread is None:
     86                          self.__shutdown_connection()
     87                          return
     88
     89                  _log.info('stopping backend notifications listener thread')
     90                  self._quit_lock.release()
     91                  try:
     92                          # give the worker thread time to terminate
     93                          self._listener_thread.join(self._poll_interval+2.0)
     94                          try:
     95                                  if self._listener_thread.isAlive():
     96                                          _log.error('listener thread still alive after join()')
     97                                          _log.debug('active threads: %s' % threading.enumerate())
     98                          except:
     99                                  pass
    100                  except:
    101                          print(sys.exc_info())
    102
    103                  self._listener_thread = None
    104
    105                  try:
    106                          self.__unregister_unspecific_notifications()
    107                  except:
    108                          _log.exception('unable to unregister unspecific notifications')
    109
    110                  self.__shutdown_connection()
    111
    112                  return
    113          #-------------------------------
    114          # event handlers
    115          #-------------------------------
    116          # internal helpers
    117          #-------------------------------
    118 -        def __register_interests(self):
    119                  # determine unspecific notifications
    120                  self.unspecific_notifications = signals2listen4
    121                  _log.info('configured unspecific notifications:')
    122                  _log.info('%s' % self.unspecific_notifications)
    123                  gmDispatcher.known_signals.extend(self.unspecific_notifications)
    124
    125                  # listen to unspecific notifications
    126                  self.__register_unspecific_notifications()
    127
    128          #-------------------------------
    129 -        def __register_unspecific_notifications(self):
    130                  for sig in self.unspecific_notifications:
    131                          _log.info('starting to listen for [%s]' % sig)
    132                          cmd = 'LISTEN "%s"' % sig
    133                          self._conn_lock.acquire(1)
    134                          try:
    135                                  self._cursor.execute(cmd)
    136                          finally:
    137                                  self._conn_lock.release()
    138
    139          #-------------------------------
    140 -        def __unregister_unspecific_notifications(self):
    141                  for sig in self.unspecific_notifications:
    142                          _log.info('stopping to listen for [%s]' % sig)
    143                          cmd = 'UNLISTEN "%s"' % sig
    144                          self._conn_lock.acquire(1)
    145                          try:
    146                                  self._cursor.execute(cmd)
    147                          finally:
    148                                  self._conn_lock.release()
    149
    150          #-------------------------------
    151 -        def __shutdown_connection(self):
    152                  _log.debug('shutting down connection with backend PID [%s]', self.backend_pid)
    153                  self._conn_lock.acquire(1)
    154                  try:
    155                          self._conn.rollback()
    156                          self._conn.close()
    157                  except:
    158                          pass                            # connection can already be closed :-(
    159                  finally:
    160                          self._conn_lock.release()
    161
    162          #-------------------------------
    163 -        def __start_thread(self):
    164                  if self._conn is None:
    165                          raise ValueError("no connection to backend available, useless to start thread")
    166
    167                  self._listener_thread = threading.Thread (
    168                          target = self._process_notifications,
    169                          name = self.__class__.__name__
    170                  )
    171                  self._listener_thread.setDaemon(True)
    172                  _log.info('starting listener thread')
    173                  self._listener_thread.start()
    174
    175          #-------------------------------
    176          # the actual thread code
    177          #-------------------------------
    178 -        def _process_notifications(self):
    179
    180                  # loop until quitting
    181                  _have_quit_lock = None
    182                  while not _have_quit_lock:
    183
    184                          # quitting ?
    185                          if self._quit_lock.acquire(0):
    186                                  break
    187
    188                          # wait at most self._poll_interval for new data
    189                          self._conn_lock.acquire(1)
    190                          try:
    191                                  ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0]
    192                          finally:
    193                                  self._conn_lock.release()
    194
    195                          # any input available ?
    196                          if len(ready_input_sockets) == 0:
    197                                  # no, select.select() timed out
    198                                  # give others a chance to grab the conn lock (eg listen/unlisten)
    199                                  time.sleep(0.3)
    200                                  continue
    201
    202                          # data available, wait for it to fully arrive
    203                          self._conn_lock.acquire(1)
    204                          try:
    205                                  self._conn.poll()
    206                          finally:
    207                                  self._conn_lock.release()
    208
    209                          # any notifications ?
    210                          while len(self._conn.notifies) > 0:
    211                                  # if self._quit_lock can be acquired we may be in
    212                                  # __del__ in which case gmDispatcher is not
    213                                  # guaranteed to exist anymore
    214                                  if self._quit_lock.acquire(0):
    215                                          _have_quit_lock = 1
    216                                          break
    217
    218                                  self._conn_lock.acquire(1)
    219                                  try:
    220                                          notification = self._conn.notifies.pop()
    221                                  finally:
    222                                          self._conn_lock.release()
    223                                  self.__notifications_received += 1
    224                                  if self.debug:
    225                                          print(notification)
    226                                  _log.debug('#%s: %s (first param is PID of sending backend)', self.__notifications_received, notification)
    227                                  # decode payload
    228                                  payload = notification.payload.split('::')
    229                                  operation = None
    230                                  table = None
    231                                  pk_column_name = None
    232                                  pk_of_row = None
    233                                  pk_identity = None
    234                                  for item in payload:
    235                                          if item.startswith('operation='):
    236                                                  operation = item.split('=')[1]
    237                                          if item.startswith('table='):
    238                                                  table = item.split('=')[1]
    239                                          if item.startswith('PK name='):
    240                                                  pk_column_name = item.split('=')[1]
    241                                          if item.startswith('row PK='):
    242                                                  pk_of_row = int(item.split('=')[1])
    243                                          if item.startswith('person PK='):
    244                                                  try:
    245                                                          pk_identity = int(item.split('=')[1])
    246                                                  except ValueError:
    247                                                          _log.exception('error in change notification trigger')
    248                                                          pk_identity = -1
    249                                  # try sending intra-client signals:
    250                                  # 1) generic signal
    251                                  self.__messages_sent += 1
    252                                  try:
    253                                          results = gmDispatcher.send (
    254                                                  signal = notification.channel,
    255                                                  originated_in_database = True,
    256                                                  listener_pid = self.backend_pid,
    257                                                  sending_backend_pid = notification.pid,
    258                                                  pk_identity = pk_identity,
    259                                                  operation = operation,
    260                                                  table = table,
    261                                                  pk_column_name = pk_column_name,
    262                                                  pk_of_row = pk_of_row,
    263                                                  message_index = self.__messages_sent,
    264                                                  notification_index = self.__notifications_received
    265                                          )
    266                                  except:
    267                                          print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid))
    268                                          print(sys.exc_info())
    269                                  # 2) dynamically emulated old style table specific signals
    270                                  if table is not None:
    271                                          self.__messages_sent += 1
    272                                          signal = '%s_mod_db' % table
    273                                          _log.debug('emulating old-style table specific signal [%s]', signal)
    274                                          try:
    275                                                  results = gmDispatcher.send (
    276                                                          signal = signal,
    277                                                          originated_in_database = True,
    278                                                          listener_pid = self.backend_pid,
    279                                                          sending_backend_pid = notification.pid,
    280                                                          pk_identity = pk_identity,
    281                                                          operation = operation,
    282                                                          table = table,
    283                                                          pk_column_name = pk_column_name,
    284                                                          pk_of_row = pk_of_row,
    285                                                          message_index = self.__messages_sent,
    286                                                          notification_index = self.__notifications_received
    287                                                  )
    288                                          except:
    289                                                  print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid))
    290                                                  print(sys.exc_info())
    291
    292                                  # there *may* be more pending notifications but
    293                                  # we don't care when quitting
    294                                  if self._quit_lock.acquire(0):
    295                                          _have_quit_lock = 1
    296                                          break
    297
    298                  # exit thread activity
    299                  return
    300  #=====================================================================
    301  # main
    302  #=====================================================================
    303  if __name__ == "__main__":
    304
    305          if len(sys.argv) < 2:
    306                  sys.exit()
    307
    308          if sys.argv[1] not in ['test', 'monitor']:
    309                  sys.exit()
    310
    311
    312          notifies = 0
    313
    314          from Gnumed.pycommon import gmPG2, gmI18N
    315          from Gnumed.business import gmPerson, gmPersonSearch
    316
    317          gmI18N.activate_locale()
    318          gmI18N.install_domain(domain='gnumed')
    319          #-------------------------------
    320 -        def run_test():
    321
    322                  #-------------------------------
    323                  def dummy(n):
    324                          return float(n)*n/float(1+n)
    325                  #-------------------------------
    326                  def OnPatientModified():
    327                          global notifies
    328                          notifies += 1
    329                          sys.stdout.flush()
    330                          print("\nBackend says: patient data has been modified (%s. notification)" % notifies)
    331                  #-------------------------------
    332                  try:
    333                          n = int(sys.argv[2])
    334                  except:
    335                          print("You can set the number of iterations\nwith the second command line argument")
    336                          n = 100000
    337
    338                  # try loop without backend listener
    339                  print("Looping", n, "times through dummy function")
    340                  i = 0
    341                  t1 = time.time()
    342                  while i < n:
    343                          r = dummy(i)
    344                          i += 1
    345                  t2 = time.time()
    346                  t_nothreads = t2-t1
    347                  print("Without backend thread, it took", t_nothreads, "seconds")
    348
    349                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    350
    351                  # now try with listener to measure impact
    352                  print("Now in a new shell connect psql to the")
    353                  print("database <gnumed_v9> on localhost, return")
    354                  print("here and hit <enter> to continue.")
    355                  input('hit <enter> when done starting psql')
    356                  print("You now have about 30 seconds to go")
    357                  print("to the psql shell and type")
    358                  print(" notify patient_changed<enter>")
    359                  print("several times.")
    360                  print("This should trigger our backend listening callback.")
    361                  print("You can also try to stop the demo with Ctrl-C !")
    362
    363                  listener.register_callback('patient_changed', OnPatientModified)
    364
    365                  try:
    366                          counter = 0
    367                          while counter < 20:
    368                                  counter += 1
    369                                  time.sleep(1)
    370                                  sys.stdout.flush()
    371                                  print('.')
    372                          print("Looping",n,"times through dummy function")
    373                          i = 0
    374                          t1 = time.time()
    375                          while i < n:
    376                                  r = dummy(i)
    377                                  i += 1
    378                          t2 = time.time()
    379                          t_threaded = t2-t1
    380                          print("With backend thread, it took", t_threaded, "seconds")
    381                          print("Difference:", t_threaded-t_nothreads)
    382                  except KeyboardInterrupt:
    383                          print("cancelled by user")
    384
    385                  listener.shutdown()
    386                  listener.unregister_callback('patient_changed', OnPatientModified)
    387          #-------------------------------
    388 -        def run_monitor():
    389
    390                  print("starting up backend notifications monitor")
    391
    392                  def monitoring_callback(*args, **kwargs):
    393                          try:
    394                                  kwargs['originated_in_database']
    395                                  print('==> got notification from database "%s":' % kwargs['signal'])
    396                          except KeyError:
    397                                  print('==> received signal from client: "%s"' % kwargs['signal'])
    398                          del kwargs['signal']
    399                          for key in kwargs.keys():
    400                                  print('    [%s]: %s' % (key, kwargs[key]))
    401
    402                  gmDispatcher.connect(receiver = monitoring_callback)
    403
    404                  listener = gmBackendListener(conn = gmPG2.get_raw_connection())
    405                  print("listening for the following notifications:")
    406                  print("1) unspecific:")
    407                  for sig in listener.unspecific_notifications:
    408                          print('   - %s' % sig)
    409
    410                  while True:
    411                          pat = gmPersonSearch.ask_for_patient()
    412                          if pat is None:
    413                                  break
    414                          print("found patient", pat)
    415                          gmPerson.set_active_patient(patient=pat)
    416                          print("now waiting for notifications, hit <ENTER> to select another patient")
    417                          input()
    418
    419                  print("cleanup")
    420                  listener.shutdown()
    421
    422                  print("shutting down backend notifications monitor")
    423
    424          #-------------------------------
    425          if sys.argv[1] == 'monitor':
    426                  run_monitor()
    427          else:
    428                  run_test()
    429
    430  #=====================================================================
    431