Gnumed.pycommon.gmPG2

Package Gnumed :: Package pycommon :: Module gmPG2 |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmPG2

       1  """GNUmed PostgreSQL connection handling.
       2
       3  TODO: iterator/generator batch fetching:
       4          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
       5          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
       6
       7  winner:
       8  def resultset_functional_batchgenerator(cursor, size=100):
       9          for results in iter(lambda: cursor.fetchmany(size), []):
      10                  for rec in results:
      11                          yield rec
      12  """
      13  # =======================================================================
      14  __author__  = "K.Hilbert <Karsten.Hilbert@gmx.net>"
      15  __license__ = 'GPL v2 or later (details at http://www.gnu.org)'
      16
      17  # stdlib
      18  import time
      19  import sys
      20  import os
      21  import stat
      22  import io
      23  import codecs
      24  import logging
      25  import datetime as pydt
      26  import re as regex
      27  import threading
      28  import hashlib
      29  import shutil
      30
      31
      32  # GNUmed
      33  if __name__ == '__main__':
      34          sys.path.insert(0, '../../')
      35  from Gnumed.pycommon import gmLoginInfo
      36  from Gnumed.pycommon import gmExceptions
      37  from Gnumed.pycommon import gmDateTime
      38  from Gnumed.pycommon import gmBorg
      39  from Gnumed.pycommon import gmI18N
      40  from Gnumed.pycommon import gmLog2
      41  from Gnumed.pycommon import gmTools
      42  from Gnumed.pycommon.gmTools import prompted_input, u_replacement_character, format_dict_like
      43
      44  _log = logging.getLogger('gm.db')
      45
      46
      47  # 3rd party
      48  try:
      49          import psycopg2 as dbapi
      50  except ImportError:
      51          _log.exception("Python database adapter psycopg2 not found.")
      52          print("CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server.")
      53          raise
      54
      55
      56  _log.info('psycopg2 version: %s' % dbapi.__version__)
      57  _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
      58  if not (float(dbapi.apilevel) >= 2.0):
      59          raise ImportError('gmPG2: supported DB-API level too low')
      60  if not (dbapi.threadsafety > 0):
      61          raise ImportError('gmPG2: lacking minimum thread safety in psycopg2')
      62  if not (dbapi.paramstyle == 'pyformat'):
      63          raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2')
      64  try:
      65          dbapi.__version__.index('dt')
      66  except ValueError:
      67          raise ImportError('gmPG2: lacking datetime support in psycopg2')
      68  try:
      69          dbapi.__version__.index('ext')
      70  except ValueError:
      71          raise ImportError('gmPG2: lacking extensions support in psycopg2')
      72  try:
      73          dbapi.__version__.index('pq3')
      74  except ValueError:
      75          raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2')
      76
      77  import psycopg2.extras
      78  import psycopg2.extensions
      79  import psycopg2.pool
      80  import psycopg2.errorcodes as sql_error_codes
      81
      82  # =======================================================================
      83  _default_client_encoding = 'UTF8'
      84  _log.info('assuming default client encoding of [%s]' % _default_client_encoding)
      85
      86  # things timezone
      87  _default_client_timezone = None                 # default time zone for connections
      88  _sql_set_timezone = None
      89  _timestamp_template = "cast('%s' as timestamp with time zone)"          # MUST NOT be uniocde or else getquoted will not work
      90  FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone
      91
      92  _default_dsn = None
      93  _default_login = None
      94
      95  default_database = 'gnumed_v22'
      96
      97  postgresql_version_string = None
      98  postgresql_version = None                       # accuracy: major.minor
      99
     100  __ro_conn_pool = None
     101
     102  auto_request_login_params = True
     103  # =======================================================================
     104  # global data
     105  # =======================================================================
     106
     107  known_schema_hashes = {
     108          0: 'not released, testing only',
     109          2: 'b09d50d7ed3f91ddf4c4ddb8ea507720',
     110          3: 'e73718eaf230d8f1d2d01afa8462e176',
     111          4: '4428ccf2e54c289136819e701bb095ea',
     112          5: '7e7b093af57aea48c288e76632a382e5',  # ... old (v1) style hashes
     113          6: '90e2026ac2efd236da9c8608b8685b2d',  # new (v2) style hashes ...
     114          7: '6c9f6d3981483f8e9433df99d1947b27',
     115          8: '89b13a7af83337c3aad153b717e52360',
     116          9: '641a9b2be3c378ffc2bb2f0b1c9f051d',
     117          10: '7ef42a8fb2bd929a2cdd0c63864b4e8a',
     118          11: '03042ae24f3f92877d986fb0a6184d76',
     119          12: '06183a6616db62257e22814007a8ed07',
     120          13: 'fab7c1ae408a6530c47f9b5111a0841e',
     121          14: 'e170d543f067d1ea60bfe9076b1560cf',
     122          15: '70012ff960b77ecdff4981c94b5b55b6',
     123          16: '0bcf44ca22c479b52976e5eda1de8161',
     124          17: '161428ee97a00e3bf56168c3a15b7b50',
     125          18: 'a0f9efcabdecfb4ddb6d8c0b69c02092',
     126          #19: '419e5225259c53dd36ad80d82066ff02' # 19.0 only
     127          #19: '9765373098b03fb208332498f34cd4b5' # until 19.11
     128          19: '57f009a159f55f77525cc0291e0c8b60', # starting with 19.12
     129          20: 'baed1901ed4c2f272b56c8cb2c6d88e8',
     130          21: 'e6a51a89dd22b75b61ead8f7083f251f',
     131          22: 'bf45f01327fb5feb2f5d3c06ba4a6792'
     132  }
     133
     134  map_schema_hash2version = {
     135          'b09d50d7ed3f91ddf4c4ddb8ea507720': 2,
     136          'e73718eaf230d8f1d2d01afa8462e176': 3,
     137          '4428ccf2e54c289136819e701bb095ea': 4,
     138          '7e7b093af57aea48c288e76632a382e5': 5,
     139          '90e2026ac2efd236da9c8608b8685b2d': 6,
     140          '6c9f6d3981483f8e9433df99d1947b27': 7,
     141          '89b13a7af83337c3aad153b717e52360': 8,
     142          '641a9b2be3c378ffc2bb2f0b1c9f051d': 9,
     143          '7ef42a8fb2bd929a2cdd0c63864b4e8a': 10,
     144          '03042ae24f3f92877d986fb0a6184d76': 11,
     145          '06183a6616db62257e22814007a8ed07': 12,
     146          'fab7c1ae408a6530c47f9b5111a0841e': 13,
     147          'e170d543f067d1ea60bfe9076b1560cf': 14,
     148          '70012ff960b77ecdff4981c94b5b55b6': 15,
     149          '0bcf44ca22c479b52976e5eda1de8161': 16,
     150          '161428ee97a00e3bf56168c3a15b7b50': 17,
     151          'a0f9efcabdecfb4ddb6d8c0b69c02092': 18,
     152          #'419e5225259c53dd36ad80d82066ff02': 19 # 19.0 only
     153          #'9765373098b03fb208332498f34cd4b5': 19 # until 19.11
     154          '57f009a159f55f77525cc0291e0c8b60': 19, # starting with 19.12
     155          'baed1901ed4c2f272b56c8cb2c6d88e8': 20,
     156          'e6a51a89dd22b75b61ead8f7083f251f': 21,
     157          'bf45f01327fb5feb2f5d3c06ba4a6792': 22
     158  }
     159
     160  map_client_branch2required_db_version = {
     161          'GIT tree': 0,
     162          '0.3': 9,
     163          '0.4': 10,
     164          '0.5': 11,
     165          '0.6': 12,
     166          '0.7': 13,
     167          '0.8': 14,
     168          '0.9': 15,
     169          '1.0': 16,              # intentional duplicate with 1.1
     170          '1.1': 16,
     171          '1.2': 17,
     172          '1.3': 18,
     173          '1.4': 19,
     174          '1.5': 20,
     175          '1.6': 21,
     176          '1.7': 22,
     177          '1.8': 22               # Yes, SAME as 1.7, no DB change.
     178  }
     179
     180  map_psyco_tx_status2str = [
     181          'TRANSACTION_STATUS_IDLE',
     182          'TRANSACTION_STATUS_ACTIVE',
     183          'TRANSACTION_STATUS_INTRANS',
     184          'TRANSACTION_STATUS_INERROR',
     185          'TRANSACTION_STATUS_UNKNOWN'
     186  ]
     187
     188  map_psyco_conn_status2str = [
     189          '0 - ?',
     190          'STATUS_READY',
     191          'STATUS_BEGIN_ALIAS_IN_TRANSACTION',
     192          'STATUS_PREPARED'
     193  ]
     194
     195  map_psyco_iso_level2str = {
     196          None: 'ISOLATION_LEVEL_DEFAULT (configured on server)',
     197          0: 'ISOLATION_LEVEL_AUTOCOMMIT',
     198          1: 'ISOLATION_LEVEL_READ_UNCOMMITTED',
     199          2: 'ISOLATION_LEVEL_REPEATABLE_READ',
     200          3: 'ISOLATION_LEVEL_SERIALIZABLE',
     201          4: 'ISOLATION_LEVEL_READ_UNCOMMITTED'
     202  }
     203
     204  # get columns and data types for a given table
     205  query_table_col_defs = """select
     206          cols.column_name,
     207          cols.udt_name
     208  from
     209          information_schema.columns cols
     210  where
     211          cols.table_schema = %s
     212                  and
     213          cols.table_name = %s
     214  order by
     215          cols.ordinal_position"""
     216
     217  query_table_attributes = """select
     218          cols.column_name
     219  from
     220          information_schema.columns cols
     221  where
     222          cols.table_schema = %s
     223                  and
     224          cols.table_name = %s
     225  order by
     226          cols.ordinal_position"""
     227
     228  # only works for single-column FKs but that's fine
     229  # needs gm-dbo, any-doc won't work
     230  SQL_foreign_key_name = """SELECT
     231          fk_tbl.*,
     232          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = fk_tbl.connamespace) AS constraint_schema,
     233          fk_tbl.conname AS constraint_name,
     234          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.conrelid)) AS source_schema,
     235          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.conrelid) AS source_table,
     236          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.conkey[1] AND attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass) AS source_column,
     237          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.confrelid)) AS target_schema,
     238          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.confrelid) AS target_table,
     239          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.confkey[1] AND attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass) AS target_column
     240  FROM
     241          pg_catalog.pg_constraint fk_tbl
     242  WHERE
     243          fk_tbl.contype = 'f'
     244                  AND
     245          fk_tbl.conrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
     246                  AND
     247          fk_tbl.conkey[1] = (
     248                  SELECT
     249                          col_tbl1.attnum
     250                  FROM
     251                          pg_catalog.pg_attribute col_tbl1
     252                  WHERE
     253                          col_tbl1.attname = %(src_col)s
     254                                  AND
     255                          col_tbl1.attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
     256          )
     257                  AND
     258          fk_tbl.confrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
     259                  AND
     260          fk_tbl.confkey[1] = (
     261                  SELECT
     262                          col_tbl2.attnum
     263                  FROM
     264                          pg_catalog.pg_attribute col_tbl2
     265                  WHERE
     266                          col_tbl2.attname = %(target_col)s
     267                                  AND
     268                          col_tbl2.attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
     269          )
     270  """
     271
     272  SQL_get_index_name = """
     273  SELECT
     274          (SELECT nspname FROM pg_namespace WHERE pg_namespace.oid = pg_class.relnamespace)
     275                  AS index_schema,
     276          pg_class.relname
     277                  AS index_name
     278  FROM
     279          pg_class
     280  WHERE
     281          pg_class.oid IN (
     282                  SELECT
     283                          indexrelid
     284                  FROM
     285                          pg_index
     286                  WHERE
     287                          pg_index.indrelid = %(idx_tbl)s::regclass
     288                                  AND
     289                          pg_index.indnatts = 1           -- only one column in index
     290                                  AND
     291                          pg_index.indkey[0] IN (
     292                                  SELECT
     293                                          pg_attribute.attnum
     294                                  FROM
     295                                          pg_attribute
     296                                  WHERE
     297                                          pg_attribute.attrelid = %(idx_tbl)s::regclass
     298                                                  AND
     299                                          pg_attribute.attname = %(idx_col)s
     300                                  )
     301          )
     302  """
     303
     304  SQL_get_pk_col_def = """
     305  SELECT
     306          pg_attribute.attname
     307                  AS pk_col,
     308          format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
     309                  AS pk_type
     310  FROM pg_index, pg_class, pg_attribute, pg_namespace
     311  WHERE
     312          pg_class.oid = %(table)s::regclass
     313                  AND
     314          indrelid = pg_class.oid
     315                  AND
     316  --      nspname = %%(schema)s
     317  --              AND
     318          pg_class.relnamespace = pg_namespace.oid
     319                  AND
     320          pg_attribute.attrelid = pg_class.oid
     321                  AND
     322          pg_attribute.attnum = any(pg_index.indkey)
     323                  AND
     324          indisprimary
     325  """
     326
     327  # =======================================================================
     328  # module globals API
     329  # =======================================================================
     330 -def set_default_client_encoding(encoding = None):
     331          # check whether psycopg2 can handle this encoding
     332          if encoding not in psycopg2.extensions.encodings:
     333                  raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding)
     334          # check whether Python can handle this encoding
     335          py_enc = psycopg2.extensions.encodings[encoding]
     336          try:
     337                  codecs.lookup(py_enc)
     338          except LookupError:
     339                  _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc))
     340                  raise
     341          # FIXME: check encoding against the database
     342          # FIXME: - but we may not yet have access
     343          # FIXME: - psycopg2 will pull its encodings from the database eventually
     344          # it seems save to set it
     345          global _default_client_encoding
     346          _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, encoding))
     347          _default_client_encoding = encoding
     348          return True
     349
     350  #---------------------------------------------------
     351 -def set_default_client_timezone(timezone = None):
     352
     353          # FIXME: use __validate
     354          global _default_client_timezone
     355          _log.info('setting default client time zone from [%s] to [%s]' % (_default_client_timezone, timezone))
     356          _default_client_timezone = timezone
     357
     358          global _sql_set_timezone
     359          _sql_set_timezone = 'set timezone to %s'
     360
     361          return True
     362
     363  #---------------------------------------------------
     364 -def __validate_timezone(conn=None, timezone=None):
     365
     366          _log.debug('validating time zone [%s]', timezone)
     367
     368          cmd = 'set timezone to %(tz)s'
     369          args = {'tz': timezone}
     370
     371          conn.commit()
     372          curs = conn.cursor()
     373          is_valid = False
     374          try:
     375                  curs.execute(cmd, args)
     376                  _log.info('time zone [%s] is settable', timezone)
     377                  # can we actually use it, though ?
     378                  cmd = """select '1920-01-19 23:00:00+01'::timestamp with time zone"""
     379                  try:
     380                          curs.execute(cmd)
     381                          curs.fetchone()
     382                          _log.info('time zone [%s] is usable', timezone)
     383                          is_valid = True
     384                  except:
     385                          _log.error('error using time zone [%s]', timezone)
     386          except dbapi.DataError:
     387                  _log.warning('time zone [%s] is not settable', timezone)
     388          except:
     389                  _log.error('failed to set time zone to [%s]', timezone)
     390                  _log.exception('')
     391
     392          curs.close()
     393          conn.rollback()
     394
     395          return is_valid
     396
     397  #---------------------------------------------------
     398 -def __expand_timezone(conn=None, timezone=None):
     399          """some timezone defs are abbreviations so try to expand
     400          them because "set time zone" doesn't take abbreviations"""
     401
     402          cmd = """
     403  select distinct on (abbrev) name
     404  from pg_timezone_names
     405  where
     406          abbrev = %(tz)s and
     407          name ~ '^[^/]+/[^/]+$' and
     408          name !~ '^Etc/'
     409  """
     410          args = {'tz': timezone}
     411
     412          conn.commit()
     413          curs = conn.cursor()
     414
     415          result = timezone
     416          try:
     417                  curs.execute(cmd, args)
     418                  rows = curs.fetchall()
     419                  if len(rows) > 0:
     420                          result = rows[0]['name']
     421                          _log.debug('[%s] maps to [%s]', timezone, result)
     422          except:
     423                  _log.exception('cannot expand timezone abbreviation [%s]', timezone)
     424
     425          curs.close()
     426          conn.rollback()
     427
     428          return result
     429
     430  #---------------------------------------------------
     431 -def __detect_client_timezone(conn=None):
     432          """This is run on the very first connection."""
     433
     434          # FIXME: check whether server.timezone is the same
     435          # FIXME: value as what we eventually detect
     436
     437          # we need gmDateTime to be initialized
     438          if gmDateTime.current_local_iso_numeric_timezone_string is None:
     439                  gmDateTime.init()
     440
     441          _log.debug('trying to detect timezone from system')
     442
     443          tz_candidates = []
     444          try:
     445                  tz = os.environ['TZ']
     446                  tz_candidates.append(tz)
     447                  expanded = __expand_timezone(conn = conn, timezone = tz)
     448                  if expanded != tz:
     449                          tz_candidates.append(expanded)
     450          except KeyError:
     451                  pass
     452
     453          tz_candidates.append(gmDateTime.current_local_timezone_name)
     454          expanded = __expand_timezone(conn = conn, timezone = gmDateTime.current_local_timezone_name)
     455          if expanded != gmDateTime.current_local_timezone_name:
     456                  tz_candidates.append(expanded)
     457
     458          _log.debug('candidates: %s', tz_candidates)
     459
     460          # find best among candidates
     461          global _default_client_timezone
     462          global _sql_set_timezone
     463          found = False
     464          for tz in tz_candidates:
     465                  if __validate_timezone(conn = conn, timezone = tz):
     466                          _default_client_timezone = tz
     467                          _sql_set_timezone = 'set timezone to %s'
     468                          found = True
     469                          break
     470
     471          if not found:
     472                  _default_client_timezone = gmDateTime.current_local_iso_numeric_timezone_string
     473                  _sql_set_timezone = "set time zone interval %s hour to minute"
     474
     475          _log.info('client system time zone detected as equivalent to [%s]', _default_client_timezone)
     476
     477  # =======================================================================
     478  # login API
     479  # =======================================================================
     480 -def __request_login_params_tui():
     481          """Text mode request of database login parameters"""
     482          import getpass
     483          login = gmLoginInfo.LoginInfo()
     484
     485          print("\nPlease enter the required login parameters:")
     486          try:
     487                  login.host = prompted_input(prompt = "host ('' = non-TCP/IP)", default = '')
     488                  login.database = prompted_input(prompt = "database", default = default_database)
     489                  login.user = prompted_input(prompt = "user name", default = '')
     490                  tmp = 'password for "%s" (not shown): ' % login.user
     491                  login.password = getpass.getpass(tmp)
     492                  gmLog2.add_word2hide(login.password)
     493                  login.port = prompted_input(prompt = "port", default = 5432)
     494          except KeyboardInterrupt:
     495                  _log.warning("user cancelled text mode login dialog")
     496                  print("user cancelled text mode login dialog")
     497                  raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!"))
     498
     499          return login
     500
     501  #---------------------------------------------------
     502 -def __request_login_params_gui_wx():
     503          """GUI (wx) input request for database login parameters.
     504
     505          Returns gmLoginInfo.LoginInfo object
     506          """
     507          import wx
     508          # OK, wxPython was already loaded. But has the main Application instance
     509          # been initialized yet ? if not, the exception will kick us out
     510          if wx.GetApp() is None:
     511                  raise AssertionError(_("The wxPython GUI framework hasn't been initialized yet!"))
     512
     513          # Let's launch the login dialog
     514          # if wx was not initialized /no main App loop, an exception should be raised anyway
     515          import gmAuthWidgets
     516          dlg = gmAuthWidgets.cLoginDialog(None, -1)
     517          dlg.ShowModal()
     518          login = dlg.panel.GetLoginInfo()
     519          dlg.Destroy()
     520
     521          #if user cancelled or something else went wrong, raise an exception
     522          if login is None:
     523                  raise gmExceptions.ConnectionError(_("Can't connect to database without login information!"))
     524
     525          gmLog2.add_word2hide(login.password)
     526
     527          return login
     528
     529  #---------------------------------------------------
     530 -def request_login_params():
     531          """Request login parameters for database connection."""
     532          # do we auto-request parameters at all ?
     533          if not auto_request_login_params:
     534                  raise Exception('Cannot request login parameters.')
     535
     536          # are we inside X ?
     537          # if we aren't wxGTK will crash hard at the C-level with "can't open Display"
     538          if 'DISPLAY' in os.environ:
     539                  # try wxPython GUI
     540                  try:
     541                          return __request_login_params_gui_wx()
     542                  except:
     543                          pass
     544
     545          # well, either we are on the console or
     546          # wxPython does not work, use text mode
     547          return __request_login_params_tui()
     548
     549  # =======================================================================
     550  # DSN API
     551  # -----------------------------------------------------------------------
     552 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
     553          dsn_parts = []
     554
     555          if (database is not None) and (database.strip() != ''):
     556                  dsn_parts.append('dbname=%s' % database)
     557
     558          if (host is not None) and (host.strip() != ''):
     559                  dsn_parts.append('host=%s' % host)
     560
     561          if (port is not None) and (str(port).strip() != ''):
     562                  dsn_parts.append('port=%s' % port)
     563
     564          if (user is not None) and (user.strip() != ''):
     565                  dsn_parts.append('user=%s' % user)
     566
     567          if (password is not None) and (password.strip() != ''):
     568                  dsn_parts.append('password=%s' % password)
     569
     570          dsn_parts.append('sslmode=prefer')
     571          dsn_parts.append('fallback_application_name=GNUmed')
     572
     573          return ' '.join(dsn_parts)
     574
     575  # ------------------------------------------------------
     576 -def get_default_login():
     577          # make sure we do have a login
     578          get_default_dsn()
     579          return _default_login
     580
     581  # ------------------------------------------------------
     582 -def get_default_dsn():
     583          global _default_dsn
     584          if _default_dsn is not None:
     585                  return _default_dsn
     586
     587          login = request_login_params()
     588          set_default_login(login=login)
     589
     590          return _default_dsn
     591
     592  # ------------------------------------------------------
     593 -def set_default_login(login=None):
     594          if login is None:
     595                  return False
     596
     597          if login.host is not None:
     598                  if login.host.strip() == '':
     599                          login.host = None
     600
     601          global _default_login
     602          _default_login = login
     603          _log.info('setting default login from [%s] to [%s]' % (_default_login, login))
     604
     605          dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password)
     606
     607          global _default_dsn
     608          if _default_dsn is None:
     609                  old_dsn = 'None'
     610          else:
     611                  old_dsn = regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, _default_dsn)
     612          _log.info ('setting default DSN from [%s] to [%s]',
     613                  old_dsn,
     614                  regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, dsn)
     615          )
     616          _default_dsn = dsn
     617
     618          return True
     619
     620  #------------------------------------------------------------------------
     621 -def log_auth_environment():
     622          try:
     623                  pgpass_file = os.path.expanduser(os.path.join('~', '.pgpass'))
     624                  if os.path.exists(pgpass_file):
     625                          _log.debug('standard .pgpass (%s) exists', pgpass_file)
     626                  else:
     627                          _log.debug('standard .pgpass (%s) not found', pgpass_file)
     628                  pgpass_var = os.getenv('PGPASSFILE')
     629                  if pgpass_var is None:
     630                          _log.debug('$PGPASSFILE not set')
     631                  else:
     632                          if os.path.exists(pgpass_var):
     633                                  _log.debug('$PGPASSFILE=%s exists', pgpass_var)
     634                          else:
     635                                  _log.debug('$PGPASSFILE=%s not found')
     636          except Exception:
     637                  _log.exception('cannot detect .pgpass and or $PGPASSFILE')
     638
     639  # =======================================================================
     640  # netadata API
     641  # =======================================================================
     642 -def database_schema_compatible(link_obj=None, version=None, verbose=True):
     643          expected_hash = known_schema_hashes[version]
     644          if version == 0:
     645                  args = {'ver': 9999}
     646          else:
     647                  args = {'ver': version}
     648          rows, idx = run_ro_queries (
     649                  link_obj = link_obj,
     650                  queries = [{
     651                          'cmd': 'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5',
     652                          'args': args
     653                  }]
     654          )
     655          if rows[0]['md5'] != expected_hash:
     656                  _log.error('database schema version mismatch')
     657                  _log.error('expected: %s (%s)' % (version, expected_hash))
     658                  _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5']))
     659                  if verbose:
     660                          _log.debug('schema dump follows:')
     661                          for line in get_schema_structure(link_obj = link_obj).split():
     662                                  _log.debug(line)
     663                          _log.debug('schema revision history dump follows:')
     664                          for line in get_schema_revision_history(link_obj = link_obj):
     665                                  _log.debug(' - '.join(line))
     666                  return False
     667          _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5']))
     668          return True
     669
     670  #------------------------------------------------------------------------
     671 -def get_schema_version(link_obj=None):
     672          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
     673          try:
     674                  return map_schema_hash2version[rows[0]['md5']]
     675          except KeyError:
     676                  return 'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
     677
     678  #------------------------------------------------------------------------
     679 -def get_schema_structure(link_obj=None):
     680          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select gm.concat_table_structure()'}])
     681          return rows[0][0]
     682
     683  #------------------------------------------------------------------------
     684 -def get_schema_hash(link_obj=None):
     685          rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
     686          return rows[0]['md5']
     687
     688  #------------------------------------------------------------------------
     689 -def get_schema_revision_history(link_obj=None):
     690
     691          if table_exists(link_obj = link_obj, schema = 'gm', table = 'schema_revision'):
     692                  cmd = """
     693                          SELECT
     694                                  imported::text,
     695                                  version,
     696                                  filename
     697                          FROM gm.schema_revision
     698                          ORDER BY imported"""
     699          elif table_exists(link_obj = link_obj, schema = 'public', table = 'gm_schema_revision'):
     700                  cmd = """
     701                          SELECT
     702                                  imported::text,
     703                                  version,
     704                                  filename
     705                          FROM public.gm_schema_revision
     706                          ORDER BY imported"""
     707          else:
     708                  return []
     709
     710          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}])
     711          return rows
     712  #------------------------------------------------------------------------
     713 -def get_current_user():
     714          rows, idx = run_ro_queries(queries = [{'cmd': 'select CURRENT_USER'}])
     715          return rows[0][0]
     716
     717  #------------------------------------------------------------------------
     718 -def get_foreign_keys2column(schema='public', table=None, column=None, link_obj=None):
     719          """Get the foreign keys pointing to schema.table.column.
     720
     721          Does not properly work with multi-column FKs.
     722          GNUmed doesn't use any, however.
     723          """
     724          args = {
     725                  'schema': schema,
     726                  'tbl': table,
     727                  'col': column
     728          }
     729          cmd = """
     730  SELECT
     731          %(schema)s AS referenced_schema,
     732          %(tbl)s AS referenced_table,
     733          %(col)s AS referenced_column,
     734          pgc.confkey AS referenced_column_list,
     735
     736          pgc.conrelid::regclass AS referencing_table,
     737          pgc.conkey AS referencing_column_list,
     738          (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) AS referencing_column
     739  FROM
     740          pg_constraint pgc
     741  WHERE
     742          pgc.contype = 'f'
     743                  AND
     744          pgc.confrelid = (
     745                  select oid from pg_class where relname = %(tbl)s and relnamespace = (
     746                          select oid from pg_namespace where nspname = %(schema)s
     747                   )
     748          )       and
     749          (
     750                  select attnum
     751                  from pg_attribute
     752                  where
     753                          attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = (
     754                                  select oid from pg_namespace where nspname = %(schema)s
     755                          ))
     756                                  and
     757                          attname = %(col)s
     758          ) = any(pgc.confkey)
     759  """
     760          rows, idx = run_ro_queries (
     761                  link_obj = link_obj,
     762                  queries = [
     763                          {'cmd': cmd, 'args': args}
     764                  ]
     765          )
     766
     767          return rows
     768
     769  #------------------------------------------------------------------------
     770 -def get_index_name(indexed_table=None, indexed_column=None, link_obj=None):
     771
     772          args = {
     773                  'idx_tbl': indexed_table,
     774                  'idx_col': indexed_column
     775          }
     776          rows, idx = run_ro_queries (
     777                  link_obj = link_obj,
     778                  queries = [{'cmd': SQL_get_index_name, 'args': args}],
     779                  get_col_idx = False
     780          )
     781
     782          return rows
     783
     784  #------------------------------------------------------------------------
     785 -def get_foreign_key_names(src_schema=None, src_table=None, src_column=None, target_schema=None, target_table=None, target_column=None, link_obj=None):
     786
     787          args = {
     788                  'src_schema': src_schema,
     789                  'src_tbl': src_table,
     790                  'src_col': src_column,
     791                  'target_schema': target_schema,
     792                  'target_tbl': target_table,
     793                  'target_col': target_column
     794          }
     795
     796          rows, idx = run_ro_queries (
     797                  link_obj = link_obj,
     798                  queries = [{'cmd': SQL_foreign_key_name, 'args': args}],
     799                  get_col_idx = False
     800          )
     801
     802          return rows
     803
     804  #------------------------------------------------------------------------
     805 -def get_child_tables(schema='public', table=None, link_obj=None):
     806          """Return child tables of <table>."""
     807          cmd = """
     808  select
     809          pgn.nspname as namespace,
     810          pgc.relname as table
     811  from
     812          pg_namespace pgn,
     813          pg_class pgc
     814  where
     815          pgc.relnamespace = pgn.oid
     816                  and
     817          pgc.oid in (
     818                  select inhrelid from pg_inherits where inhparent = (
     819                          select oid from pg_class where
     820                                  relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
     821                                  relname = %(table)s
     822                  )
     823          )"""
     824          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}])
     825          return rows
     826
     827  #------------------------------------------------------------------------
     828 -def schema_exists(link_obj=None, schema='gm'):
     829          cmd = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = %(schema)s)"""
     830          args = {'schema': schema}
     831          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
     832          return rows[0][0]
     833
     834  #------------------------------------------------------------------------
     835 -def table_exists(link_obj=None, schema=None, table=None):
     836          """Returns false, true."""
     837          cmd = """
     838  select exists (
     839          select 1 from information_schema.tables
     840          where
     841                  table_schema = %s and
     842                  table_name = %s and
     843                  table_type = 'BASE TABLE'
     844  )"""
     845          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}])
     846          return rows[0][0]
     847
     848  #------------------------------------------------------------------------
     849 -def function_exists(link_obj=None, schema=None, function=None):
     850
     851          cmd = """
     852                  SELECT EXISTS (
     853                          SELECT 1 FROM pg_proc
     854                          WHERE proname = %(func)s AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = %(schema)s)
     855                  )
     856          """
     857          args = {
     858                  'func': function,
     859                  'schema': schema
     860          }
     861          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
     862          return rows[0][0]
     863
     864  #------------------------------------------------------------------------
     865 -def get_col_indices(cursor = None):
     866          if cursor.description is None:
     867                  _log.error('no result description available: unused cursor or last query did not select rows')
     868                  return None
     869          col_indices = {}
     870          col_index = 0
     871          for col_desc in cursor.description:
     872                  col_name = col_desc[0]
     873                  # a query like "select 1,2;" will return two columns of the same name !
     874                  # hence adjust to that, note, however, that dict-style access won't work
     875                  # on results of such queries ...
     876                  if col_name in col_indices:
     877                          col_name = '%s_%s' % (col_name, col_index)
     878                  col_indices[col_name] = col_index
     879                  col_index += 1
     880
     881          return col_indices
     882  #------------------------------------------------------------------------
     883 -def get_col_defs(link_obj=None, schema='public', table=None):
     884          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}])
     885          col_names = []
     886          col_type = {}
     887          for row in rows:
     888                  col_names.append(row[0])
     889                  # map array types
     890                  if row[1].startswith('_'):
     891                          col_type[row[0]] = row[1][1:] + '[]'
     892                  else:
     893                          col_type[row[0]] = row[1]
     894          col_defs = []
     895          col_defs.append(col_names)
     896          col_defs.append(col_type)
     897          return col_defs
     898  #------------------------------------------------------------------------
     899 -def get_col_names(link_obj=None, schema='public', table=None):
     900          """Return column attributes of table"""
     901          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}])
     902          cols = []
     903          for row in rows:
     904                  cols.append(row[0])
     905          return cols
     906
     907  #------------------------------------------------------------------------
     908  # i18n functions
     909  #------------------------------------------------------------------------
     910 -def export_translations_from_database(filename=None):
     911          tx_file = io.open(filename, mode = 'wt', encoding = 'utf8')
     912          tx_file.write('-- GNUmed database string translations exported %s\n' % gmDateTime.pydt_now_here().strftime('%Y-%m-%d %H:%M'))
     913          tx_file.write('-- - contains translations for each of [%s]\n' % ', '.join(get_translation_languages()))
     914          tx_file.write('-- - user database language is set to [%s]\n\n' % get_current_user_language())
     915          tx_file.write('-- Please email this file to <gnumed-devel@gnu.org>.\n')
     916          tx_file.write('-- ----------------------------------------------------------------------------------------------\n\n')
     917          tx_file.write('set default_transaction_read_only to off;\n\n')
     918          tx_file.write("set client_encoding to 'utf-8';\n\n")
     919          tx_file.write('\\unset ON_ERROR_STOP\n\n')
     920
     921          cmd = 'SELECT lang, orig, trans FROM i18n.translations ORDER BY lang, orig'
     922          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
     923          for row in rows:
     924                  line = "select i18n.upd_tx(E'%s', E'%s', E'%s');\n" % (
     925                          row['lang'].replace("'", "\\'"),
     926                          row['orig'].replace("'", "\\'"),
     927                          row['trans'].replace("'", "\\'")
     928                  )
     929                  tx_file.write(line)
     930          tx_file.write('\n')
     931
     932          tx_file.write('\set ON_ERROR_STOP 1\n')
     933          tx_file.close()
     934
     935          return True
     936
     937  #------------------------------------------------------------------------
     938 -def delete_translation_from_database(link_obj=None, language=None, original=None):
     939          cmd = 'DELETE FROM i18n.translations WHERE lang = %(lang)s AND orig = %(orig)s'
     940          args = {'lang': language, 'orig': original}
     941          run_rw_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}], return_data = False, end_tx = True)
     942          return True
     943
     944  #------------------------------------------------------------------------
     945 -def update_translation_in_database(language=None, original=None, translation=None, link_obj=None):
     946          if language is None:
     947                  cmd = 'SELECT i18n.upd_tx(%(orig)s, %(trans)s)'
     948          else:
     949                  cmd = 'SELECT i18n.upd_tx(%(lang)s, %(orig)s, %(trans)s)'
     950          args = {'lang': language, 'orig': original, 'trans': translation}
     951          run_rw_queries(queries = [{'cmd': cmd, 'args': args}], return_data = False, link_obj = link_obj)
     952          return args
     953
     954  #------------------------------------------------------------------------
     955 -def get_translation_languages():
     956          rows, idx = run_ro_queries (
     957                  queries = [{'cmd': 'select distinct lang from i18n.translations'}]
     958          )
     959          return [ r[0] for r in rows ]
     960
     961  #------------------------------------------------------------------------
     962 -def get_database_translations(language=None, order_by=None):
     963
     964          args = {'lang': language}
     965          _log.debug('language [%s]', language)
     966
     967          if order_by is None:
     968                  order_by = 'ORDER BY %s' % order_by
     969          else:
     970                  order_by = 'ORDER BY lang, orig'
     971
     972          if language is None:
     973                  cmd = """
     974                  SELECT DISTINCT ON (orig, lang)
     975                          lang, orig, trans
     976                  FROM ((
     977
     978                          -- strings stored as translation keys whether translated or not
     979                          SELECT
     980                                  NULL as lang,
     981                                  ik.orig,
     982                                  NULL AS trans
     983                          FROM
     984                                  i18n.keys ik
     985
     986                  ) UNION ALL (
     987
     988                          -- already translated strings
     989                          SELECT
     990                                  it.lang,
     991                                  it.orig,
     992                                  it.trans
     993                          FROM
     994                                  i18n.translations it
     995
     996                  )) as translatable_strings
     997                  %s""" % order_by
     998          else:
     999                  cmd = """
    1000                  SELECT DISTINCT ON (orig, lang)
    1001                          lang, orig, trans
    1002                  FROM ((
    1003
    1004                          -- strings stored as translation keys whether translated or not
    1005                          SELECT
    1006                                  %%(lang)s as lang,
    1007                                  ik.orig,
    1008                                  i18n._(ik.orig, %%(lang)s) AS trans
    1009                          FROM
    1010                                  i18n.keys ik
    1011
    1012                  ) UNION ALL (
    1013
    1014                          -- already translated strings
    1015                          SELECT
    1016                                  %%(lang)s as lang,
    1017                                  it.orig,
    1018                                  i18n._(it.orig, %%(lang)s) AS trans
    1019                          FROM
    1020                                  i18n.translations it
    1021
    1022                  )) AS translatable_strings
    1023                  %s""" % order_by
    1024
    1025          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
    1026
    1027          if rows is None:
    1028                  _log.error('no translatable strings found')
    1029          else:
    1030                  _log.debug('%s translatable strings found', len(rows))
    1031
    1032          return rows
    1033
    1034  #------------------------------------------------------------------------
    1035 -def get_current_user_language():
    1036          cmd = 'select i18n.get_curr_lang()'
    1037          rows, idx = run_ro_queries(queries = [{'cmd': cmd}])
    1038          return rows[0][0]
    1039
    1040  #------------------------------------------------------------------------
    1041 -def set_user_language(user=None, language=None):
    1042          """Set the user language in the database.
    1043
    1044          user = None: current db user
    1045          language = None: unset
    1046          """
    1047          _log.info('setting database language for user [%s] to [%s]', user, language)
    1048
    1049          args = {
    1050                  'usr': user,
    1051                  'lang': language
    1052          }
    1053
    1054          if language is None:
    1055                  if user is None:
    1056                          queries = [{'cmd': 'select i18n.unset_curr_lang()'}]
    1057                  else:
    1058                          queries = [{'cmd': 'select i18n.unset_curr_lang(%(usr)s)', 'args': args}]
    1059                  queries.append({'cmd': 'select True'})
    1060          else:
    1061                  if user is None:
    1062                          queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s)', 'args': args}]
    1063                  else:
    1064                          queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}]
    1065
    1066          rows, idx = run_rw_queries(queries = queries, return_data = True)
    1067
    1068          if not rows[0][0]:
    1069                  _log.error('cannot set database language to [%s] for user [%s]', language, user)
    1070
    1071          return rows[0][0]
    1072  #------------------------------------------------------------------------
    1073 -def force_user_language(language=None):
    1074          """Set the user language in the database.
    1075
    1076          - regardless of whether there is any translation available.
    1077          - only for the current user
    1078          """
    1079          _log.info('forcing database language for current db user to [%s]', language)
    1080
    1081          run_rw_queries(queries = [{
    1082                  'cmd': 'select i18n.force_curr_lang(%(lang)s)',
    1083                  'args': {'lang': language}
    1084          }])
    1085
    1086  # =======================================================================
    1087  # query runners and helpers
    1088  # =======================================================================
    1089 -def send_maintenance_notification():
    1090          cmd = 'notify "db_maintenance_warning"'
    1091          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
    1092
    1093  #------------------------------------------------------------------------
    1094 -def send_maintenance_shutdown():
    1095          cmd = 'notify "db_maintenance_disconnect"'
    1096          run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
    1097
    1098  #------------------------------------------------------------------------
    1099 -def is_pg_interval(candidate=None):
    1100          cmd = 'SELECT %(candidate)s::interval'
    1101          try:
    1102                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
    1103                  return True
    1104          except:
    1105                  cmd = 'SELECT %(candidate)s::text::interval'
    1106                  try:
    1107                          rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
    1108                          return True
    1109                  except:
    1110                          return False
    1111
    1112  #------------------------------------------------------------------------
    1113 -def lock_row(link_obj=None, table=None, pk=None, exclusive=False):
    1114          """Uses pg_advisory(_shared).
    1115
    1116          - locks stack upon each other and need one unlock per lock
    1117          - same connection:
    1118                  - all locks succeed
    1119          - different connections:
    1120                  - shared + shared succeed
    1121                  - shared + exclusive fail
    1122          """
    1123          _log.debug('locking row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1124          if exclusive:
    1125                  cmd = """SELECT pg_try_advisory_lock('%s'::regclass::oid::int, %s)""" % (table, pk)
    1126          else:
    1127                  cmd = """SELECT pg_try_advisory_lock_shared('%s'::regclass::oid::int, %s)""" % (table, pk)
    1128          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
    1129          if rows[0][0]:
    1130                  return True
    1131          _log.warning('cannot lock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1132          return False
    1133
    1134  #------------------------------------------------------------------------
    1135 -def unlock_row(link_obj=None, table=None, pk=None, exclusive=False):
    1136          """Uses pg_advisory_unlock(_shared).
    1137
    1138          - each lock needs one unlock
    1139          """
    1140          _log.debug('trying to unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1141          if exclusive:
    1142                  cmd = "SELECT pg_advisory_unlock('%s'::regclass::oid::int, %s)" % (table, pk)
    1143          else:
    1144                  cmd = "SELECT pg_advisory_unlock_shared('%s'::regclass::oid::int, %s)" % (table, pk)
    1145          rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
    1146          if rows[0][0]:
    1147                  return True
    1148          _log.warning('cannot unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
    1149          return False
    1150
    1151  #------------------------------------------------------------------------
    1152 -def row_is_locked(table=None, pk=None):
    1153          """Looks at pk_locks
    1154
    1155          - does not take into account locks other than 'advisory', however
    1156          """
    1157          cmd = """SELECT EXISTS (
    1158                  SELECT 1 FROM pg_locks WHERE
    1159                          classid = '%s'::regclass::oid::int
    1160                                  AND
    1161                          objid = %s
    1162                                  AND
    1163                          locktype = 'advisory'
    1164          )""" % (table, pk)
    1165          rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
    1166          if rows[0][0]:
    1167                  _log.debug('row is locked: [%s] [%s]', table, pk)
    1168                  return True
    1169          _log.debug('row is NOT locked: [%s] [%s]', table, pk)
    1170          return False
    1171
    1172  #------------------------------------------------------------------------
    1173 -def __store_file_in_cache(filename, cache_key_data):
    1174
    1175          md5 = hashlib.md5()
    1176          md5.update(('%s' % cache_key_data).encode('utf8'))
    1177          md5_sum = md5.hexdigest()
    1178          cached_name = os.path.join(gmTools.gmPaths().bytea_cache_dir, md5_sum)
    1179          _log.debug('caching [%s] as [%s]', filename, cached_name)
    1180          gmTools.remove_file(cached_name, log_error = True, force = True)
    1181          try:
    1182                  shutil.copyfile(filename, cached_name, follow_symlinks = True)
    1183          except shutil.SameFileError:
    1184                  pass
    1185          except OSError:
    1186                  _log.exception('cannot copy file into cache: [%s] -> [%s]', filename, cached_name)
    1187                  return None
    1188          PERMS_owner_only = 0o0660
    1189          try:
    1190                  os.chmod(cached_name, PERMS_owner_only)
    1191          except PermissionError:
    1192                  _log.exception('cannot set cache file [%s] permissions to [%s]', cached_name, stat.filemode(PERMS_owner_only))
    1193                  return None
    1194          return cached_name
    1195
    1196  #------------------------------------------------------------------------
    1197 -def __get_filename_in_cache(cache_key_data=None, data_size=None):
    1198
    1199          md5 = hashlib.md5()
    1200          md5.update(('%s' % cache_key_data).encode('utf8'))
    1201          md5_sum = md5.hexdigest()
    1202          cached_name = os.path.join(gmTools.gmPaths().bytea_cache_dir, md5_sum)
    1203          _log.debug('[%s]: %s', md5_sum, cached_name)
    1204          try:
    1205                  stat = os.stat(cached_name)
    1206          except FileNotFoundError:
    1207                  return None
    1208          _log.debug('cache hit: %s [%s]', cached_name, stat)
    1209          if os.path.islink(cached_name) or (not os.path.isfile(cached_name)):
    1210                  _log.error('object in cache is not a regular file: %s', cached_name)
    1211                  _log.error('possibly an attack, removing')
    1212                  removed = gmTools.remove_file(cached_name, log_error = True)
    1213                  if removed:
    1214                          return None
    1215                  raise BaseException('cannot delete suspicious object in cache dir: %s', cached_name)
    1216          if stat.st_size == data_size:
    1217                  return cached_name
    1218          _log.debug('size in cache [%s] <> expected size [%s], removing cached file', stat.st_size, data_size)
    1219          removed = gmTools.remove_file(cached_name, log_error = True)
    1220          if removed:
    1221                  return None
    1222          raise BaseException('cannot remove suspicous object from cache dir: %s', cached_name)
    1223
    1224  #------------------------------------------------------------------------
    1225 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None, conn=None, link2cached=True):
    1226
    1227          if data_size is None:
    1228                  rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
    1229                  data_size = rows[0][0]
    1230                  if data_size in [None, 0]:
    1231                          conn.rollback()
    1232                          return True
    1233
    1234          cache_key_data = '%s' % data_query
    1235          cached_filename = __get_filename_in_cache(cache_key_data = cache_key_data, data_size = data_size)
    1236          if cached_filename is not None:
    1237                  # link to desired name
    1238                  if link2cached:
    1239                          if gmTools.mklink(cached_filename, filename, overwrite = False):
    1240                                  return True
    1241                          _log.debug('cannot link to cache, trying copy-from-cache')
    1242                  # copy from cache
    1243                  try:
    1244                          shutil.copyfile(cached_filename, filename, follow_symlinks = True)
    1245                          return True
    1246                  except shutil.SameFileError:
    1247                          pass
    1248                  except OSError:
    1249                          _log.exception('cannot copy file from cache: [%s] -> [%s]', cached_filename, filename)
    1250                  # if cache fails entirely -> fall through to new file
    1251                  _log.debug('downloading new copy of file, despite found in cache')
    1252
    1253          outfile = io.open(filename, 'wb')
    1254          result = bytea2file_object (
    1255                  data_query = data_query,
    1256                  file_obj = outfile,
    1257                  chunk_size = chunk_size,
    1258                  data_size = data_size,
    1259                  data_size_query = data_size_query,
    1260                  conn = conn
    1261          )
    1262          outfile.close()
    1263          __store_file_in_cache(filename, cache_key_data)
    1264
    1265          return result
    1266
    1267  #------------------------------------------------------------------------
    1268 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None, conn=None):
    1269          """Store data from a bytea field into a file.
    1270
    1271          <data_query>
    1272          - dict {'cmd': ..., 'args': ...}
    1273          - 'cmd' must be a string containing "... substring(data from %(start)s for %(size)s) ..."
    1274          - 'args' must be a dict
    1275          - must return one row with one field of type bytea
    1276          <file>
    1277          - must be a file like Python object
    1278          <data_size>
    1279          - integer of the total size of the expected data or None
    1280          <data_size_query>
    1281          - dict {'cmd': ..., 'args': ...}
    1282          - must return one row with one field with the octet_length() of the data field
    1283          - used only when <data_size> is None
    1284          """
    1285          if data_size == 0:
    1286                  return True
    1287
    1288          # If the client sets an encoding other than the default we
    1289          # will receive encoding-parsed data which isn't the binary
    1290          # content we want. Hence we need to get our own connection.
    1291          # It must be a read-write one so that we don't affect the
    1292          # encoding for other users of the shared read-only
    1293          # connections.
    1294          # Actually, encodings shouldn't be applied to binary data
    1295          # (eg. bytea types) in the first place but that is only
    1296          # reported to be fixed > v7.4.
    1297          # further tests reveal that at least on PG 8.0 this bug still
    1298          # manifests itself
    1299          if conn is None:
    1300                  conn = get_raw_connection(readonly = True)
    1301
    1302          if data_size is None:
    1303                  rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
    1304                  data_size = rows[0][0]
    1305                  if data_size in [None, 0]:
    1306                          conn.rollback()
    1307                          return True
    1308
    1309          max_chunk_size = 1024 * 1024 * 20                       # 20 MB, works for typical CR DICOMs
    1310          if chunk_size == 0:
    1311                  chunk_size = min(data_size, max_chunk_size)
    1312
    1313          _log.debug('expecting %s bytes of BYTEA data in chunks of %s bytes', data_size, chunk_size)
    1314
    1315          # Windoze sucks: it can't transfer objects of arbitrary size,
    1316          # anyways, we need to split the transfer,
    1317          # however, only possible if postgres >= 7.2
    1318          needed_chunks, remainder = divmod(data_size, chunk_size)
    1319          _log.debug('# of chunks: %s; remainder: %s bytes', needed_chunks, remainder)
    1320
    1321          # retrieve chunks, skipped if data size < chunk size,
    1322          # does this not carry the danger of cutting up multi-byte escape sequences ?
    1323          # no, since bytea is binary,
    1324          # yes, since in bytea there are *some* escaped values, still
    1325          # no, since those are only escaped during *transfer*, not on-disk, hence
    1326          # only complete escape sequences are put on the wire
    1327          for chunk_id in range(needed_chunks):
    1328                  chunk_start = (chunk_id * chunk_size) + 1
    1329                  data_query['args']['start'] = chunk_start
    1330                  data_query['args']['size'] = chunk_size
    1331                  try:
    1332                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1333                  except:
    1334                          _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size))
    1335                          conn.rollback()
    1336                          raise
    1337                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1338                  file_obj.write(rows[0][0])
    1339
    1340          # retrieve remainder
    1341          if remainder > 0:
    1342                  chunk_start = (needed_chunks * chunk_size) + 1
    1343                  data_query['args']['start'] = chunk_start
    1344                  data_query['args']['size'] = remainder
    1345                  try:
    1346                          rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
    1347                  except:
    1348                          _log.error('cannot retrieve remaining [%s] bytes' % remainder)
    1349                          conn.rollback()
    1350                          raise
    1351                  # it would be a fatal error to see more than one result as ids are supposed to be unique
    1352                  file_obj.write(rows[0][0])
    1353
    1354          conn.rollback()
    1355          return True
    1356
    1357  #------------------------------------------------------------------------
    1358 -def file2bytea(query=None, filename=None, args=None, conn=None, file_md5=None):
    1359          """Store data from a file into a bytea field.
    1360
    1361          The query must:
    1362          - be in unicode
    1363          - contain a format spec identifying the row (eg a primary key)
    1364            matching <args> if it is an UPDATE
    1365          - contain a format spec " <field> = %(data)s::bytea"
    1366
    1367          The query CAN return the MD5 of the inserted data:
    1368                  RETURNING md5(<field>) AS md5
    1369          in which case it will compare it to the md5
    1370          of the file.
    1371          """
    1372          # read data from file
    1373          infile = open(filename, "rb")
    1374          data_as_byte_string = infile.read()
    1375          infile.close()
    1376          if args is None:
    1377                  args = {}
    1378          # really still needed for BYTEA input ?
    1379          args['data'] = memoryview(data_as_byte_string)
    1380          del(data_as_byte_string)
    1381
    1382          # insert the data
    1383          if conn is None:
    1384                  conn = get_raw_connection(readonly = False)
    1385                  close_conn = True
    1386          else:
    1387                  close_conn = False
    1388
    1389          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = (file_md5 is not None))
    1390
    1391          success_status = True
    1392          if file_md5 is None:
    1393                  conn.commit()
    1394          else:
    1395                  db_md5 = rows[0]['md5']
    1396                  if file_md5 != db_md5:
    1397                          conn.rollback()
    1398                          success_status = False
    1399                          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1400                  else:
    1401                          conn.commit()
    1402                          _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1403
    1404          if close_conn:
    1405                  conn.close()
    1406
    1407          return success_status
    1408
    1409  #------------------------------------------------------------------------
    1410 -def file2lo(filename=None, conn=None, check_md5=False):
    1411          # 1 GB limit unless 64 bit Python build ...
    1412          file_size = os.path.getsize(filename)
    1413          if file_size > (1024 * 1024) * 1024:
    1414                  _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
    1415  #               return -1
    1416
    1417          if conn is None:
    1418                  conn = get_raw_connection(readonly = False)
    1419                  close_conn = conn.close
    1420          else:
    1421                  close_conn = __noop
    1422          _log.debug('[%s] -> large object', filename)
    1423
    1424          # insert the data
    1425          lo = conn.lobject(0, 'w', 0, filename)
    1426          lo_oid = lo.oid
    1427          lo.close()
    1428          _log.debug('large object OID: %s', lo_oid)
    1429
    1430          # verify
    1431          if file_md5 is None:
    1432                  conn.commit()
    1433                  close_conn()
    1434                  return lo_oid
    1435          cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
    1436          args = {'loid': lo_oid}
    1437          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
    1438          db_md5 = rows[0][0]
    1439          if file_md5 == db_md5:
    1440                  conn.commit()
    1441                  close_conn()
    1442                  _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1443                  return lo_oid
    1444          conn.rollback()
    1445          close_conn()
    1446          _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
    1447          return -1
    1448
    1449  #------------------------------------------------------------------------
    1450 -def file2bytea_lo(filename=None, conn=None, file_md5=None):
    1451          # 1 GB limit unless 64 bit Python build ...
    1452          file_size = os.path.getsize(filename)
    1453          if file_size > (1024 * 1024) * 1024:
    1454                  _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
    1455  #               return -1
    1456
    1457          if conn is None:
    1458                  conn = get_raw_connection(readonly = False)
    1459                  close_conn = conn.close
    1460          else:
    1461                  close_conn = __noop
    1462          _log.debug('[%s] -> large object', filename)
    1463
    1464          # insert the data
    1465          lo = conn.lobject(0, 'w', 0, filename)
    1466          lo_oid = lo.oid
    1467          lo.close()
    1468          _log.debug('large object OID: %s', lo_oid)
    1469
    1470          # verify
    1471          if file_md5 is None:
    1472                  conn.commit()
    1473                  close_conn()
    1474                  return lo_oid
    1475          cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
    1476          args = {'loid': lo_oid}
    1477          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
    1478          db_md5 = rows[0][0]
    1479          if file_md5 == db_md5:
    1480                  conn.commit()
    1481                  close_conn()
    1482                  _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1483                  return lo_oid
    1484          conn.rollback()
    1485          close_conn()
    1486          _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
    1487          return -1
    1488
    1489  #------------------------------------------------------------------------
    1490 -def file2bytea_copy_from(table=None, columns=None, filename=None, conn=None, md5_query=None, file_md5=None):
    1491          # md5_query: dict{'cmd': ..., 'args': ...}
    1492
    1493          # UNTESTED
    1494
    1495          chunk_size = 32 * (1024 * 1024)
    1496          _log.debug('[%s] (%s bytes) --(%s bytes)-> %s(%s)', filename, os.path.getsize(filename), chunk_size, table, columns)
    1497          if conn is None:
    1498                  conn = get_raw_connection(readonly = False)
    1499                  close_conn = True
    1500          else:
    1501                  close_conn = False
    1502          curs = conn.cursor()
    1503          # write
    1504          infile = open(filename, "rb")
    1505          curs.copy_from(infile, table, size = chunk_size, columns = columns)
    1506          infile.close()
    1507          curs.close()
    1508          if None in [file_md5, md5_query]:
    1509                  conn.commit()
    1510                  close_conn()
    1511                  return True
    1512          # verify
    1513          rows, idx = run_ro_queries(link_obj = conn, queries = [md5_query])
    1514          db_md5 = rows[0][0]
    1515          if file_md5 == db_md5:
    1516                  conn.commit()
    1517                  close_conn()
    1518                  _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1519                  return True
    1520          close_conn()
    1521          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1522          return False
    1523
    1524  #------------------------------------------------------------------------
    1525 -def file2bytea_overlay(query=None, args=None, filename=None, conn=None, md5_query=None, file_md5=None):
    1526          """Store data from a file into a bytea field.
    1527
    1528          The query must:
    1529          - 'cmd' must be in unicode
    1530          - 'cmd' must contain a format spec identifying the row (eg
    1531            a primary key) matching <args> if it is an UPDATE
    1532          - 'cmd' must contain "... SET ... <some_bytea_field> = OVERLAY(some_bytea_field PLACING %(data)s::bytea FROM %(start)s FOR %(size)s) ..."
    1533          - 'args' must be a dict matching 'cmd'
    1534
    1535          The query CAN return the MD5 of the inserted data:
    1536                  RETURNING md5(<field>) AS md5
    1537          in which case it will compare it to the md5
    1538          of the file.
    1539
    1540          UPDATE
    1541                  the_table
    1542          SET
    1543                  bytea_field = OVERLAY (
    1544                          coalesce(bytea_field, '':bytea),
    1545                          PLACING
    1546                                  %(data)s::bytea
    1547                          FROM
    1548                                  %(start)s
    1549                          FOR
    1550                                  %(size)s
    1551                  )
    1552          WHERE
    1553                  primary_key = pk_value
    1554
    1555          SELECT md5(bytea_field) FROM the_table WHERE primary_key = pk_value
    1556          """
    1557          chunk_size = 32 * (1024 * 1024)
    1558          file_size = os.path.getsize(filename)
    1559          if file_size <= chunk_size:
    1560                  chunk_size = file_size
    1561          needed_chunks, remainder = divmod(file_size, chunk_size)
    1562          _log.debug('file data: %s bytes, chunks: %s, chunk size: %s bytes, remainder: %s bytes', file_size, needed_chunks, chunk_size, remainder)
    1563
    1564          if conn is None:
    1565                  conn = get_raw_connection(readonly = False)
    1566                  close_conn = conn.close
    1567          else:
    1568                  close_conn = __noop
    1569
    1570          infile = open(filename, "rb")
    1571          # write chunks
    1572          for chunk_id in range(needed_chunks):
    1573                  chunk_start = (chunk_id * chunk_size) + 1
    1574                  args['start'] = chunk_start
    1575                  args['size'] = chunk_size
    1576                  data_as_byte_string = infile.read(chunk_size)
    1577                  # really still needed for BYTEA input ?
    1578                  args['data'] = memoryview(data_as_byte_string)
    1579                  del(data_as_byte_string)
    1580                  try:
    1581                          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
    1582                  except Exception:
    1583                          _log.exception('cannot write chunk [%s/%s] of size [%s], try decreasing chunk size', chunk_id+1, needed_chunks, chunk_size)
    1584                          conn.rollback()
    1585                          close_conn()
    1586                          infile.close()
    1587                          raise
    1588          # write remainder
    1589          if remainder > 0:
    1590                  chunk_start = (needed_chunks * chunk_size) + 1
    1591                  args['start'] = chunk_start
    1592                  args['size'] = remainder
    1593                  data_as_byte_string = infile.read(remainder)
    1594                  # really still needed for BYTEA input ?
    1595                  args['data'] = memoryview(data_as_byte_string)
    1596                  del(data_as_byte_string)
    1597                  try:
    1598                          rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
    1599                  except Exception:
    1600                          _log.error('cannot retrieve remaining [%s] bytes' % remainder)
    1601                          conn.rollback()
    1602                          close_conn()
    1603                          infile.close()
    1604                          raise
    1605          infile.close()
    1606          if None in [file_md5, md5_query]:
    1607                  conn.commit()
    1608                  close_conn()
    1609                  return True
    1610          # verify
    1611          rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': md5_query, 'args': args}])
    1612          db_md5 = rows[0][0]
    1613          if file_md5 == db_md5:
    1614                  conn.commit()
    1615                  close_conn()
    1616                  _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
    1617                  return True
    1618          close_conn()
    1619          _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
    1620          return False
    1621
    1622  #---------------------------------------------------------------------------
    1623 -def run_sql_script(sql_script, conn=None):
    1624
    1625          if conn is None:
    1626                  conn = get_connection(readonly = False)
    1627
    1628          from Gnumed.pycommon import gmPsql
    1629          psql = gmPsql.Psql(conn)
    1630
    1631          if psql.run(sql_script) == 0:
    1632                  query = {
    1633                          'cmd': 'select gm.log_script_insertion(%(name)s, %(ver)s)',
    1634                          'args': {'name': sql_script, 'ver': 'current'}
    1635                  }
    1636                  run_rw_queries(link_obj = conn, queries = [query])
    1637                  conn.commit()
    1638                  return True
    1639
    1640          _log.error('error running sql script: %s', sql_script)
    1641          return False
    1642
    1643  #------------------------------------------------------------------------
    1644 -def sanitize_pg_regex(expression=None, escape_all=False):
    1645          """Escape input for use in a PostgreSQL regular expression.
    1646
    1647          If a fragment comes from user input and is to be used
    1648          as a regular expression we need to make sure it doesn't
    1649          contain invalid regex patterns such as unbalanced ('s.
    1650
    1651          <escape_all>
    1652                  True: try to escape *all* metacharacters
    1653                  False: only escape those which render the regex invalid
    1654          """
    1655          return expression.replace (
    1656                          '(', '\('
    1657                  ).replace (
    1658                          ')', '\)'
    1659                  ).replace (
    1660                          '[', '\['
    1661                  ).replace (
    1662                          '+', '\+'
    1663                  ).replace (
    1664                          '.', '\.'
    1665                  ).replace (
    1666                          '*', '\*'
    1667                  )
    1668                  #']', '\]',                     # not needed
    1669
    1670  #------------------------------------------------------------------------
    1671 -def capture_conn_state(conn=None):
    1672
    1673          tx_status = conn.get_transaction_status()
    1674          if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
    1675                  isolation_level = '%s (tx aborted or unknown, cannot retrieve)' % conn.isolation_level
    1676          else:
    1677                  isolation_level = '%s (%s)' % (conn.isolation_level, map_psyco_iso_level2str[conn.isolation_level])
    1678          conn_status = '%s (%s)' % (conn.status, map_psyco_conn_status2str[conn.status])
    1679          if conn.closed != 0:
    1680                  conn_status = 'undefined (%s)' % conn_status
    1681          try:
    1682                  conn_deferrable = conn.deferrable
    1683          except AttributeError:
    1684                  conn_deferrable = 'unavailable'
    1685
    1686          d = {
    1687                  'identity': id(conn),
    1688                  'backend PID': conn.get_backend_pid(),
    1689                  'protocol version': conn.protocol_version,
    1690                  'encoding': conn.encoding,
    1691                  'closed': conn.closed,
    1692                  'readonly': conn.readonly,
    1693                  'autocommit': conn.autocommit,
    1694                  'isolation level (psyco)': isolation_level,
    1695                  'async': conn.async_,
    1696                  'deferrable': conn_deferrable,
    1697                  'transaction status': '%s (%s)' % (tx_status, map_psyco_tx_status2str[tx_status]),
    1698                  'connection status': conn_status,
    1699                  'executing async op': conn.isexecuting(),
    1700                  'type': type(conn)
    1701          }
    1702          return '%s\n' % conn + format_dict_like (
    1703                  d,
    1704                  relevant_keys = [
    1705                          'type',
    1706                          'identity',
    1707                          'backend PID',
    1708                          'protocol version',
    1709                          'encoding',
    1710                          'isolation level (psyco)',
    1711                          'readonly',
    1712                          'autocommit',
    1713                          'closed',
    1714                          'connection status',
    1715                          'transaction status',
    1716                          'deferrable',
    1717                          'async',
    1718                          'executing async op'
    1719                  ],
    1720                  tabular = True,
    1721                  value_delimiters = None
    1722          )
    1723
    1724  #------------------------------------------------------------------------
    1725 -def capture_cursor_state(cursor=None):
    1726          conn = cursor.connection
    1727
    1728          tx_status = conn.get_transaction_status()
    1729          if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
    1730                  isolation_level = 'tx aborted or unknown, cannot retrieve'
    1731          else:
    1732                  isolation_level = conn.isolation_level
    1733          try:
    1734                  conn_deferrable = conn.deferrable
    1735          except AttributeError:
    1736                  conn_deferrable = 'unavailable'
    1737
    1738          if cursor.query is None:
    1739                  query = '<no query>'
    1740          else:
    1741                  #query = str(cursor.query, 'utf8', 'replace')
    1742                  query = cursor.query
    1743
    1744          txt = """Link state:
    1745  Cursor
    1746    identity: %s; name: %s
    1747    closed: %s; scrollable: %s; with hold: %s; arraysize: %s; itersize: %s;
    1748    last rowcount: %s; rownumber: %s; lastrowid (OID): %s;
    1749    last description: %s
    1750    statusmessage: %s
    1751  Connection
    1752    identity: %s; backend pid: %s; protocol version: %s;
    1753    closed: %s; autocommit: %s; isolation level: %s; encoding: %s; async: %s; deferrable: %s; readonly: %s;
    1754    TX status: %s; CX status: %s; executing async op: %s;
    1755  Query
    1756    %s
    1757  """ % (
    1758                  id(cursor),
    1759                  cursor.name,
    1760                  cursor.closed,
    1761                  cursor.scrollable,
    1762                  cursor.withhold,
    1763                  cursor.arraysize,
    1764                  cursor.itersize,
    1765                  cursor.rowcount,
    1766                  cursor.rownumber,
    1767                  cursor.lastrowid,
    1768                  cursor.description,
    1769                  cursor.statusmessage,
    1770
    1771                  id(conn),
    1772                  conn.get_backend_pid(),
    1773                  conn.protocol_version,
    1774                  conn.closed,
    1775                  conn.autocommit,
    1776                  isolation_level,
    1777                  conn.encoding,
    1778                  conn.async_,
    1779                  conn_deferrable,
    1780                  conn.readonly,
    1781                  map_psyco_tx_status2str[tx_status],
    1782                  map_psyco_conn_status2str[conn.status],
    1783                  conn.isexecuting(),
    1784
    1785                  query
    1786          )
    1787          return txt
    1788
    1789  #------------------------------------------------------------------------
    1790 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
    1791          """Run read-only queries.
    1792
    1793          <queries> must be a list of dicts:
    1794                  [
    1795                          {'cmd': <string>, 'args': <dict> or <tuple>},
    1796                          {...},
    1797                          ...
    1798                  ]
    1799          """
    1800          if isinstance(link_obj, dbapi._psycopg.cursor):
    1801                  curs = link_obj
    1802                  curs_close = lambda :1
    1803                  tx_rollback = lambda :1
    1804                  readonly_rollback_just_in_case = lambda :1
    1805          elif isinstance(link_obj, dbapi._psycopg.connection):
    1806                  curs = link_obj.cursor()
    1807                  curs_close = curs.close
    1808                  tx_rollback = link_obj.rollback
    1809                  if link_obj.autocommit is True:         # readonly connection ?
    1810                          readonly_rollback_just_in_case = link_obj.rollback
    1811                  else:
    1812                          # do not rollback readonly queries on passed-in readwrite
    1813                          # connections just in case because they may have already
    1814                          # seen fully legitimate write action which would get lost
    1815                          readonly_rollback_just_in_case = lambda :1
    1816          elif link_obj is None:
    1817                  conn = get_connection(readonly=True, verbose=verbose)
    1818                  curs = conn.cursor()
    1819                  curs_close = curs.close
    1820                  tx_rollback = conn.rollback
    1821                  readonly_rollback_just_in_case = conn.rollback
    1822          else:
    1823                  raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
    1824
    1825          if verbose:
    1826                  _log.debug('cursor: %s', curs)
    1827
    1828          for query in queries:
    1829                  try:
    1830                          args = query['args']
    1831                  except KeyError:
    1832                          args = None
    1833                  try:
    1834                          curs.execute(query['cmd'], args)
    1835                          if verbose:
    1836                                  _log.debug(capture_cursor_state(curs))
    1837                  except dbapi.Error as pg_exc:
    1838                          _log.error('query failed in RO connection')
    1839                          _log.error(capture_cursor_state(curs))
    1840                          if hasattr(pg_exc, 'diag'):
    1841                                  for prop in dir(pg_exc.diag):
    1842                                          if prop.startswith('__'):
    1843                                                  continue
    1844                                          val = getattr(pg_exc.diag, prop)
    1845                                          if val is None:
    1846                                                  continue
    1847                                          _log.error('PG diags %s: %s', prop, val)
    1848                          pg_exc = make_pg_exception_fields_unicode(pg_exc)
    1849                          _log.error('PG error code: %s', pg_exc.pgcode)
    1850                          if pg_exc.pgerror is not None:
    1851                                  _log.error('PG error message: %s', pg_exc.u_pgerror)
    1852                          try:
    1853                                  curs_close()
    1854                          except dbapi.InterfaceError:
    1855                                  _log.exception('cannot close cursor')
    1856                          tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1857                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    1858                                  details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
    1859                                  if curs.statusmessage != '':
    1860                                          details = 'Status: %s\n%s' % (
    1861                                                  curs.statusmessage.strip().strip('\n').strip().strip('\n'),
    1862                                                  details
    1863                                          )
    1864                                  if pg_exc.pgerror is None:
    1865                                          msg = '[%s]' % pg_exc.pgcode
    1866                                  else:
    1867                                          msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.u_pgerror)
    1868                                  raise gmExceptions.AccessDenied (
    1869                                          msg,
    1870                                          source = 'PostgreSQL',
    1871                                          code = pg_exc.pgcode,
    1872                                          details = details
    1873                                  )
    1874                          raise
    1875                  except:
    1876                          _log.exception('query failed in RO connection')
    1877                          _log.error(capture_cursor_state(curs))
    1878                          try:
    1879                                  curs_close()
    1880                          except dbapi.InterfaceError:
    1881                                  _log.exception('cannot close cursor')
    1882                          tx_rollback()           # need to rollback so ABORT state isn't preserved in pooled conns
    1883                          raise
    1884
    1885          data = None
    1886          col_idx = None
    1887          if return_data:
    1888                  data = curs.fetchall()
    1889                  if verbose:
    1890                          _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data))
    1891                          _log.debug('cursor description: %s', curs.description)
    1892                  if get_col_idx:
    1893                          col_idx = get_col_indices(curs)
    1894
    1895          curs_close()
    1896          # so we can see data committed meanwhile if the
    1897          # link object had been passed in and thusly might
    1898          # be part of a long-running read-only transaction
    1899          readonly_rollback_just_in_case()
    1900          return (data, col_idx)
    1901
    1902  #------------------------------------------------------------------------
    1903 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
    1904          """Convenience function for running a transaction
    1905             that is supposed to get committed.
    1906
    1907          <link_obj>
    1908                  can be either:
    1909                  - a cursor
    1910                  - a connection
    1911
    1912          <queries>
    1913                  is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>)
    1914                  to be executed as a single transaction, the last
    1915                  query may usefully return rows (such as a
    1916                  "SELECT currval('some_sequence')" statement)
    1917
    1918          <end_tx>
    1919                  - controls whether the transaction is finalized (eg.
    1920                    committed/rolled back) or not, this allows the
    1921                    call to run_rw_queries() to be part of a framing
    1922                    transaction
    1923                  - if link_obj is a connection then <end_tx> will
    1924                    default to False unless it is explicitly set to
    1925                    True which is taken to mean "yes, you do have full
    1926                    control over the transaction" in which case the
    1927                    transaction is properly finalized
    1928                  - if link_obj is a cursor we CANNOT finalize the
    1929                    transaction because we would need the connection for that
    1930                  - if link_obj is None <end_tx> will, of course, always be True
    1931
    1932          <return_data>
    1933                  - if true, the returned data will include the rows
    1934                    the last query selected
    1935                  - if false, it returns None instead
    1936
    1937          <get_col_idx>
    1938                  - if true, the returned data will include a dictionary
    1939                    mapping field names to column positions
    1940                  - if false, the returned data returns None instead
    1941
    1942          method result:
    1943                  - returns a tuple (data, idx)
    1944                  - <data>:
    1945                          * (None, None) if last query did not return rows
    1946                          * ("fetchall() result", <index>) if last query returned any rows
    1947                          * for <index> see <get_col_idx>
    1948          """
    1949          if isinstance(link_obj, dbapi._psycopg.cursor):
    1950                  conn_close = lambda :1
    1951                  conn_commit = lambda :1
    1952                  tx_rollback = lambda :1
    1953                  curs = link_obj
    1954                  curs_close = lambda :1
    1955                  notices_accessor = curs.connection
    1956          elif isinstance(link_obj, dbapi._psycopg.connection):
    1957                  conn_close = lambda :1
    1958                  if end_tx:
    1959                          conn_commit = link_obj.commit
    1960                          tx_rollback = link_obj.rollback
    1961                  else:
    1962                          conn_commit = lambda :1
    1963                          tx_rollback = lambda :1
    1964                  curs = link_obj.cursor()
    1965                  curs_close = curs.close
    1966                  notices_accessor = link_obj
    1967          elif link_obj is None:
    1968                  conn = get_connection(readonly=False)
    1969                  conn_close = conn.close
    1970                  conn_commit = conn.commit
    1971                  tx_rollback = conn.rollback
    1972                  curs = conn.cursor()
    1973                  curs_close = curs.close
    1974                  notices_accessor = conn
    1975          else:
    1976                  raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
    1977
    1978          for query in queries:
    1979                  try:
    1980                          args = query['args']
    1981                  except KeyError:
    1982                          args = None
    1983                  try:
    1984                          curs.execute(query['cmd'], args)
    1985                          if verbose:
    1986                                  _log.debug(capture_cursor_state(curs))
    1987                          for notice in notices_accessor.notices:
    1988                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    1989                          del notices_accessor.notices[:]
    1990                  # DB related exceptions
    1991                  except dbapi.Error as pg_exc:
    1992                          _log.error('query failed in RW connection')
    1993                          _log.error(capture_cursor_state(curs))
    1994                          if hasattr(pg_exc, 'diag'):
    1995                                  for prop in dir(pg_exc.diag):
    1996                                          if prop.startswith('__'):
    1997                                                  continue
    1998                                          val = getattr(pg_exc.diag, prop)
    1999                                          if val is None:
    2000                                                  continue
    2001                                          _log.error('PG diags %s: %s', prop, val)
    2002                          for notice in notices_accessor.notices:
    2003                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    2004                          del notices_accessor.notices[:]
    2005                          pg_exc = make_pg_exception_fields_unicode(pg_exc)
    2006                          _log.error('PG error code: %s', pg_exc.pgcode)
    2007                          if pg_exc.pgerror is not None:
    2008                                  _log.error('PG error message: %s', pg_exc.u_pgerror)
    2009                          # privilege problem
    2010                          if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
    2011                                  details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
    2012                                  if curs.statusmessage != '':
    2013                                          details = 'Status: %s\n%s' % (
    2014                                                  curs.statusmessage.strip().strip('\n').strip().strip('\n'),
    2015                                                  details
    2016                                          )
    2017                                  if pg_exc.pgerror is None:
    2018                                          msg = '[%s]' % pg_exc.pgcode
    2019                                  else:
    2020                                          msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.u_pgerror)
    2021                                  try:
    2022                                          curs_close()
    2023                                          tx_rollback()                   # just for good measure
    2024                                          conn_close()
    2025                                  except dbapi.InterfaceError:
    2026                                          _log.exception('cannot cleanup')
    2027                                  raise gmExceptions.AccessDenied (
    2028                                          msg,
    2029                                          source = 'PostgreSQL',
    2030                                          code = pg_exc.pgcode,
    2031                                          details = details
    2032                                  )
    2033                          # other problem
    2034                          gmLog2.log_stack_trace()
    2035                          try:
    2036                                  curs_close()
    2037                                  tx_rollback()                   # just for good measure
    2038                                  conn_close()
    2039                          except dbapi.InterfaceError:
    2040                                  _log.exception('cannot cleanup')
    2041                          raise
    2042                  # other exception
    2043                  except:
    2044                          _log.exception('error running query in RW connection')
    2045                          _log.error(capture_cursor_state(curs))
    2046                          for notice in notices_accessor.notices:
    2047                                  _log.debug(notice.replace('\n', '/').replace('\n', '/'))
    2048                          del notices_accessor.notices[:]
    2049                          gmLog2.log_stack_trace()
    2050                          try:
    2051                                  curs_close()
    2052                                  tx_rollback()
    2053                                  conn_close()
    2054                          except dbapi.InterfaceError:
    2055                                  _log.exception('cannot cleanup')
    2056                          raise
    2057
    2058          data = None
    2059          col_idx = None
    2060          if return_data:
    2061                  try:
    2062                          data = curs.fetchall()
    2063                  except:
    2064                          _log.exception('error fetching data from RW query')
    2065                          gmLog2.log_stack_trace()
    2066                          try:
    2067                                  curs_close()
    2068                                  tx_rollback()
    2069                                  conn_close()
    2070                          except dbapi.InterfaceError:
    2071                                  _log.exception('cannot cleanup')
    2072                                  raise
    2073                          raise
    2074                  if get_col_idx:
    2075                          col_idx = get_col_indices(curs)
    2076
    2077          curs_close()
    2078          conn_commit()
    2079          conn_close()
    2080
    2081          return (data, col_idx)
    2082
    2083  #------------------------------------------------------------------------
    2084 -def run_insert(link_obj=None, schema=None, table=None, values=None, returning=None, end_tx=False, get_col_idx=False, verbose=False):
    2085          """Generates SQL for an INSERT query.
    2086
    2087          values: dict of values keyed by field to insert them into
    2088          """
    2089          if schema is None:
    2090                  schema = 'public'
    2091
    2092          fields = values.keys()          # that way val_snippets and fields really should end up in the same order
    2093          val_snippets = []
    2094          for field in fields:
    2095                  val_snippets.append('%%(%s)s' % field)
    2096
    2097          if returning is None:
    2098                  returning = ''
    2099                  return_data = False
    2100          else:
    2101                  returning = '\n\tRETURNING\n\t\t%s' % ', '.join(returning)
    2102                  return_data = True
    2103
    2104          cmd = """\nINSERT INTO %s.%s (
    2105                  %s
    2106          ) VALUES (
    2107                  %s
    2108          )%s""" % (
    2109                  schema,
    2110                  table,
    2111                  ',\n\t\t'.join(fields),
    2112                  ',\n\t\t'.join(val_snippets),
    2113                  returning
    2114          )
    2115
    2116          _log.debug('running SQL: >>>%s<<<', cmd)
    2117
    2118          return run_rw_queries (
    2119                  link_obj = link_obj,
    2120                  queries = [{'cmd': cmd, 'args': values}],
    2121                  end_tx = end_tx,
    2122                  return_data = return_data,
    2123                  get_col_idx = get_col_idx,
    2124                  verbose = verbose
    2125          )
    2126
    2127  # =======================================================================
    2128  # connection handling API
    2129  # -----------------------------------------------------------------------
    2130 -class cConnectionPool(psycopg2.pool.PersistentConnectionPool):
    2131          """GNUmed database connection pool.
    2132
    2133          Extends psycopg2's ThreadedConnectionPool with
    2134          a custom _connect() function. Supports one connection
    2135          per thread - which also ties it to one particular DSN."""
    2136          #--------------------------------------------------
    2137 -        def _connect(self, key=None):
    2138                  _log.debug('conn request with key [%s]', key)
    2139                  conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly = True)
    2140                  # monkey patching close()
    2141                  conn.original_close = conn.close
    2142                  conn.close = _raise_exception_on_ro_conn_close
    2143                  if key is not None:
    2144                          self._used[key] = conn
    2145                          self._rused[id(conn)] = key
    2146                  else:
    2147                          self._pool.append(conn)
    2148                  return conn
    2149
    2150          #--------------------------------------------------
    2151 -        def discard_connection(self, key=None):
    2152                  if key is None:
    2153                          key = threading.current_thread().ident
    2154                  try:
    2155                          conn = self._used[key]
    2156                  except KeyError:
    2157                          _log.error('no such key in connection pool: %s', key)
    2158                          _log.debug('available keys: %s', self._used.keys())
    2159                          return
    2160                  del self._used[key]
    2161                  del self._rused[id(conn)]
    2162                  conn.original_close()
    2163
    2164          #--------------------------------------------------
    2165 -        def shutdown(self):
    2166                  for conn_key in self._used.keys():
    2167                          conn = self._used[conn_key]
    2168                          if conn.closed != 0:
    2169                                  continue
    2170                          _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid())
    2171                          conn.original_close()
    2172
    2173  # -----------------------------------------------------------------------
    2174 -def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=None, autocommit=False):
    2175          """Get a raw, unadorned connection.
    2176
    2177          - this will not set any parameters such as encoding, timezone, datestyle
    2178          - the only requirement is a valid DSN
    2179          - hence it can be used for "service" connections
    2180            for verifying encodings etc
    2181          """
    2182          # FIXME: support verbose
    2183          if dsn is None:
    2184                  dsn = get_default_dsn()
    2185
    2186          if 'host=salaam.homeunix' in dsn:
    2187                  raise ValueError('The public database is not hosted by <salaam.homeunix.com> anymore.\n\nPlease point your configuration files to <publicdb.gnumed.de>.')
    2188
    2189          # try to enforce a useful encoding early on so that we
    2190          # have a good chance of decoding authentication errors
    2191          # containing foreign language characters
    2192          if ' client_encoding=' not in dsn:
    2193                  dsn += ' client_encoding=utf8'
    2194
    2195          if ' application_name' not in dsn:
    2196                  if connection_name is None:
    2197                          dsn += " application_name=GNUmed-[%s]"  % threading.current_thread().name.replace(' ', '_')
    2198                  else:
    2199                          dsn += " application_name=%s" % connection_name
    2200
    2201          try:
    2202                  # DictConnection now _is_ a real dictionary
    2203                  conn = dbapi.connect(dsn = dsn, connection_factory = psycopg2.extras.DictConnection)
    2204          except dbapi.OperationalError as e:
    2205                  t, v, tb = sys.exc_info()
    2206                  try:
    2207                          msg = e.args[0]
    2208                  except (AttributeError, IndexError, TypeError):
    2209                          raise
    2210                  if 'fe_sendauth' in msg:
    2211                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2212                  if regex.search('user ".*" does not exist', msg) is not None:
    2213                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2214                  if ((   (regex.search('user ".*"', msg) is not None)
    2215                                          or
    2216                                  (regex.search('(R|r)ol{1,2}e', msg) is not None)
    2217                          )
    2218                          and ('exist' in msg)
    2219                          and (regex.search('n(o|ich)t', msg) is not None)
    2220                  ):
    2221                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2222                  if regex.search('user ".*" does not exist', msg) is not None:
    2223                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2224                  if 'uthenti' in msg:
    2225                          raise cAuthenticationError(dsn, msg).with_traceback(tb)
    2226                  raise
    2227
    2228          if connection_name is None:
    2229                  _log.debug('established anonymous database connection, backend PID: %s', conn.get_backend_pid())
    2230          else:
    2231                  _log.debug('established database connection "%s", backend PID: %s', connection_name, conn.get_backend_pid())
    2232
    2233          # do first-connection-only stuff
    2234          # - verify PG version
    2235          global postgresql_version
    2236          if postgresql_version is None:
    2237                  curs = conn.cursor()
    2238                  curs.execute("""
    2239                          SELECT
    2240                                  substring(setting, E'^\\\\d{1,2}\\\\.\\\\d{1,2}')::numeric AS version
    2241                          FROM
    2242                                  pg_settings
    2243                          WHERE
    2244                                  name = 'server_version'
    2245                  """)
    2246                  postgresql_version = curs.fetchone()['version']
    2247                  _log.info('PostgreSQL version (numeric): %s' % postgresql_version)
    2248                  try:
    2249                          curs.execute("SELECT pg_size_pretty(pg_database_size(current_database()))")
    2250                          _log.info('database size: %s', curs.fetchone()[0])
    2251                  except BaseException:
    2252                          _log.exception('cannot get database size')
    2253                  finally:
    2254                          curs.close()
    2255                          conn.commit()
    2256                  if verbose:
    2257                          curs = conn.cursor()
    2258                          _log_PG_settings(curs = curs)
    2259                          curs.close()
    2260          # - verify PG understands client time zone
    2261          if _default_client_timezone is None:
    2262                  __detect_client_timezone(conn = conn)
    2263
    2264          # - set access mode
    2265          if readonly:
    2266                  _log.debug('readonly: forcing autocommit=True to avoid <IDLE IN TRANSACTION>')
    2267                  autocommit = True
    2268          else:
    2269                  _log.debug('autocommit is desired to be: %s', autocommit)
    2270
    2271          conn.commit()
    2272          conn.autocommit = autocommit
    2273          conn.readonly = readonly
    2274
    2275          # - assume verbose=True to mean we want debugging in the database, too
    2276          if verbose:
    2277                  _log.debug('enabling <plpgsql.extra_warnings/_errors>')
    2278                  curs = conn.cursor()
    2279                  try:
    2280                          curs.execute("SET plpgsql.extra_warnings TO 'all'")
    2281                          curs.execute("SET plpgsql.extra_errors TO 'all'")
    2282                  except BaseException:
    2283                          _log.exception('cannot enable <plpgsql.extra_warnings/_errors>')
    2284                  finally:
    2285                          curs.close()
    2286                          conn.commit()
    2287
    2288          conn.is_decorated = False
    2289          return conn
    2290
    2291  # =======================================================================
    2292 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True, connection_name=None, autocommit=False):
    2293          """Get a new connection.
    2294
    2295          This assumes the locale system has been initialized
    2296          unless an encoding is specified.
    2297          """
    2298          # FIXME: support pooled on RW, too
    2299          # FIXME: for now, support the default DSN only
    2300          if pooled and readonly and (dsn is None):
    2301                  global __ro_conn_pool
    2302                  if __ro_conn_pool is None:
    2303                          log_ro_conn = True
    2304                          __ro_conn_pool = cConnectionPool (
    2305                                  minconn = 1,
    2306                                  maxconn = 2,
    2307                                  dsn = dsn,
    2308                                  verbose = verbose
    2309                          )
    2310                  else:
    2311                          log_ro_conn = False
    2312                  try:
    2313                          conn = __ro_conn_pool.getconn()
    2314                  except psycopg2.pool.PoolError:
    2315                          _log.exception('falling back to non-pooled connection')
    2316                          conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
    2317                          log_ro_conn = True
    2318                  if log_ro_conn:
    2319                          [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
    2320          else:
    2321                  conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
    2322
    2323          if conn.is_decorated:
    2324                  return conn
    2325
    2326          if encoding is None:
    2327                  encoding = _default_client_encoding
    2328          if encoding is None:
    2329                  encoding = gmI18N.get_encoding()
    2330                  _log.warning('client encoding not specified')
    2331                  _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding)
    2332                  _log.warning('for this to work properly the application MUST have called locale.setlocale() before')
    2333
    2334          # set connection properties
    2335          # - client encoding
    2336          try:
    2337                  conn.set_client_encoding(encoding)
    2338          except dbapi.DataError:
    2339                  t, v, tb = sys.exc_info()
    2340  #               if str(v).find('cannot set encoding to') != -1:
    2341                  if 'cannot set encoding to' in str(v):
    2342                          raise cEncodingError(encoding, v).with_traceback(tb)
    2343                  if 'invalid value for parameter "client_encoding"' in str(v):
    2344                          raise cEncodingError(encoding, v).with_traceback(tb)
    2345                  raise
    2346
    2347          # - transaction isolation level
    2348          if readonly:
    2349                  # alter-database default, checked at connect, no need to set here
    2350                  pass
    2351          else:
    2352                  conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE)
    2353
    2354          _log.debug('client time zone [%s]', _default_client_timezone)
    2355
    2356          # - client time zone
    2357          curs = conn.cursor()
    2358          curs.execute(_sql_set_timezone, [_default_client_timezone])
    2359          curs.close()
    2360          conn.commit()
    2361
    2362          conn.is_decorated = True
    2363
    2364          if verbose:
    2365                  [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
    2366
    2367          return conn
    2368
    2369  #-----------------------------------------------------------------------
    2370 -def discard_pooled_connection(conn_key=None):
    2371          if __ro_conn_pool is None:
    2372                  return
    2373          __ro_conn_pool.discard_connection(key = conn_key)
    2374
    2375  #-----------------------------------------------------------------------
    2376 -def shutdown():
    2377          if __ro_conn_pool is None:
    2378                  return
    2379          __ro_conn_pool.shutdown()
    2380
    2381  # ======================================================================
    2382  # internal helpers
    2383  #-----------------------------------------------------------------------
    2384 -def __noop():
    2385          pass
    2386
    2387  #-----------------------------------------------------------------------
    2388 -def _raise_exception_on_ro_conn_close():
    2389          raise TypeError('close() called on read-only connection')
    2390
    2391  #-----------------------------------------------------------------------
    2392 -def log_database_access(action=None):
    2393          run_insert (
    2394                  schema = 'gm',
    2395                  table = 'access_log',
    2396                  values = {'user_action': action},
    2397                  end_tx = True
    2398          )
    2399
    2400  #-----------------------------------------------------------------------
    2401 -def sanity_check_time_skew(tolerance=60):
    2402          """Check server time and local time to be within
    2403          the given tolerance of each other.
    2404
    2405          tolerance: seconds
    2406          """
    2407          _log.debug('maximum skew tolerance (seconds): %s', tolerance)
    2408
    2409          cmd = "SELECT now() at time zone 'UTC'"
    2410          conn = get_raw_connection(readonly=True)
    2411          curs = conn.cursor()
    2412
    2413          start = time.time()
    2414          rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}])
    2415          end = time.time()
    2416          client_now_as_utc = pydt.datetime.utcnow()
    2417
    2418          curs.close()
    2419          conn.commit()
    2420
    2421          server_now_as_utc = rows[0][0]
    2422          query_duration = end - start
    2423          _log.info('server "now" (UTC): %s', server_now_as_utc)
    2424          _log.info('client "now" (UTC): %s', client_now_as_utc)
    2425          _log.debug('wire roundtrip (seconds): %s', query_duration)
    2426
    2427          if query_duration > tolerance:
    2428                  _log.error('useless to check client/server time skew, wire roundtrip > tolerance')
    2429                  return False
    2430
    2431          if server_now_as_utc > client_now_as_utc:
    2432                  real_skew = server_now_as_utc - client_now_as_utc
    2433          else:
    2434                  real_skew = client_now_as_utc - server_now_as_utc
    2435
    2436          _log.debug('client/server time skew: %s', real_skew)
    2437
    2438          if real_skew > pydt.timedelta(seconds = tolerance):
    2439                  _log.error('client/server time skew > tolerance')
    2440                  return False
    2441
    2442          return True
    2443
    2444  #-----------------------------------------------------------------------
    2445 -def sanity_check_database_settings():
    2446          """Checks database settings.
    2447
    2448          returns (status, message)
    2449          status:
    2450                  0: no problem
    2451                  1: non-fatal problem
    2452                  2: fatal problem
    2453          """
    2454          _log.debug('checking database settings')
    2455
    2456          conn = get_connection()
    2457
    2458          # - version string
    2459          global postgresql_version_string
    2460          if postgresql_version_string is None:
    2461                  curs = conn.cursor()
    2462                  curs.execute('SELECT version()')
    2463                  postgresql_version_string = curs.fetchone()['version']
    2464                  curs.close()
    2465                  _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string)
    2466
    2467          options2check = {
    2468                  # setting: [expected value, risk, fatal?]
    2469                  'allow_system_table_mods': [['off'], 'system breakage', False],
    2470                  'check_function_bodies': [['on'], 'suboptimal error detection', False],
    2471                  'datestyle': [['ISO'], 'faulty timestamp parsing', True],
    2472                  'default_transaction_isolation': [['read committed'], 'faulty database reads', True],
    2473                  'default_transaction_read_only': [['on'], 'accidental database writes', False],
    2474                  'fsync': [['on'], 'data loss/corruption', True],
    2475                  'full_page_writes': [['on'], 'data loss/corruption', False],
    2476                  'lc_messages': [['C'], 'suboptimal error detection', False],
    2477                  'password_encryption': [['on', 'md5', 'scram-sha-256'], 'breach of confidentiality', False],
    2478                  #u'regex_flavor': [[u'advanced'], u'query breakage', False],                                    # 9.0 doesn't support this anymore, default now advanced anyway
    2479                  'synchronous_commit': [['on'], 'data loss/corruption', False],
    2480                  'sql_inheritance': [['on'], 'query breakage, data loss/corruption', True],      # IF returned (<PG10): better be ON, if NOT returned (PG10): hardwired
    2481                  'ignore_checksum_failure': [['off'], 'data loss/corruption', False],            # starting with PG 9.3
    2482                  'track_commit_timestamp': [['on'], 'suboptimal auditing', False]                        # starting with PG 9.3
    2483          }
    2484
    2485          from Gnumed.pycommon import gmCfg2
    2486          _cfg = gmCfg2.gmCfgData()
    2487          if _cfg.get(option = 'hipaa'):
    2488                  options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', True]
    2489                  options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', True]
    2490          else:
    2491                  options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', None]
    2492                  options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', None]
    2493
    2494          cmd = "SELECT name, setting from pg_settings where name in %(settings)s"
    2495          rows, idx = run_ro_queries (
    2496                  link_obj = conn,
    2497                  queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}],
    2498                  get_col_idx = False
    2499          )
    2500
    2501          found_error = False
    2502          found_problem = False
    2503          msg = []
    2504          for row in rows:
    2505                  option = row['name']
    2506                  value_found = row['setting']
    2507                  values_expected = options2check[option][0]
    2508                  risk = options2check[option][1]
    2509                  fatal_setting = options2check[option][2]
    2510                  if value_found not in values_expected:
    2511                          if fatal_setting is True:
    2512                                  found_error = True
    2513                          elif fatal_setting is False:
    2514                                  found_problem = True
    2515                          elif fatal_setting is None:
    2516                                  pass
    2517                          else:
    2518                                  _log.error(options2check[option])
    2519                                  raise ValueError('invalid database configuration sanity check')
    2520                          msg.append(_(' option [%s]: %s') % (option, value_found))
    2521                          msg.append(_('  risk: %s') % risk)
    2522                          _log.warning('PG option [%s] set to [%s], expected %s, risk: <%s>' % (option, value_found, values_expected, risk))
    2523
    2524          if found_error:
    2525                  return 2, '\n'.join(msg)
    2526
    2527          if found_problem:
    2528                  return 1, '\n'.join(msg)
    2529
    2530          return 0, ''
    2531
    2532  #------------------------------------------------------------------------
    2533 -def _log_PG_settings(curs=None):
    2534          # don't use any of the run_*()s helper functions
    2535          # since that might create a loop if we fail here
    2536          try:
    2537                  # .pending_restart does not exist in PG 9.4 yet
    2538                  #curs.execute(u'SELECT name, setting, unit, source, reset_val, sourcefile, sourceline, pending_restart FROM pg_settings')
    2539                  curs.execute('SELECT name, setting, unit, source, reset_val, sourcefile, sourceline FROM pg_settings')
    2540          except:
    2541                  _log.exception('cannot log PG settings ("SELECT ... FROM pg_settings" failed)')
    2542                  return False
    2543          settings = curs.fetchall()
    2544          for setting in settings:
    2545                  if setting['unit'] is None:
    2546                          unit = ''
    2547                  else:
    2548                          unit = ' %s' % setting['unit']
    2549                  if setting['sourcefile'] is None:
    2550                          sfile = ''
    2551                  else:
    2552                          sfile = '// %s @ %s' % (setting['sourcefile'], setting['sourceline'])
    2553  #               # .pending_restart does not exist in PG 9.4 yet
    2554  #               if setting['pending_restart'] is False:
    2555  #                       pending_restart = u''
    2556  #               else:
    2557  #                       pending_restart = u'// needs restart'
    2558  #               _log.debug(u'%s: %s%s (set from: [%s] // sess RESET will set to: [%s]%s%s)',
    2559                  _log.debug('%s: %s%s (set from: [%s] // sess RESET will set to: [%s]%s)',
    2560                          setting['name'],
    2561                          setting['setting'],
    2562                          unit,
    2563                          setting['source'],
    2564                          setting['reset_val'],
    2565  #                       pending_restart,
    2566                          sfile
    2567                  )
    2568
    2569          try:
    2570                  curs.execute('select pg_available_extensions()')
    2571          except:
    2572                  _log.exception('cannot log available PG extensions')
    2573                  return False
    2574          extensions = curs.fetchall()
    2575          if extensions is None:
    2576                  _log.error('no PG extensions available')
    2577                  return False
    2578          for ext in extensions:
    2579                  _log.debug('PG extension: %s', ext['pg_available_extensions'])
    2580
    2581          # not really that useful because:
    2582          # - clusterwide
    2583          # - not retained across server restart (fixed in 9.6.1 - really ?)
    2584  #       try:
    2585  #               curs.execute(u'SELECT pg_last_committed_xact()')
    2586  #       except:
    2587  #               _log.exception(u'cannot retrieve last committed xact')
    2588  #       xact = curs.fetchall()
    2589  #       if xact is not None:
    2590  #               _log.debug(u'last committed transaction in cluster: %s', xact[0])
    2591
    2592          return True
    2593
    2594  #========================================================================
    2595 -def make_pg_exception_fields_unicode(exc):
    2596
    2597          if not isinstance(exc, dbapi.Error):
    2598                  return exc
    2599
    2600          if exc.pgerror is None:
    2601                  try:
    2602                          msg = exc.args[0]
    2603                  except (AttributeError, IndexError, TypeError):
    2604                          return exc
    2605                  # assumption
    2606                  exc.u_pgerror = msg
    2607                  return exc
    2608
    2609          # assumption
    2610          exc.u_pgerror = exc.pgerror.strip().strip('\n').strip().strip('\n')
    2611
    2612          return exc
    2613
    2614  #------------------------------------------------------------------------
    2615 -def extract_msg_from_pg_exception(exc=None):
    2616
    2617          try:
    2618                  return '%s' % exc.args[0]
    2619          except (AttributeError, IndexError, TypeError):
    2620                  return 'cannot extract message from exception'
    2621
    2622  # =======================================================================
    2623 -class cAuthenticationError(dbapi.OperationalError):
    2624
    2625 -        def __init__(self, dsn=None, prev_val=None):
    2626                  self.dsn = dsn
    2627                  self.prev_val = prev_val
    2628
    2629 -        def __str__(self):
    2630                  return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
    2631
    2632  # =======================================================================
    2633  # custom psycopg2 extensions
    2634  # =======================================================================
    2635 -class cEncodingError(dbapi.OperationalError):
    2636
    2637 -        def __init__(self, encoding=None, prev_val=None):
    2638                  self.encoding = encoding
    2639                  self.prev_val = prev_val
    2640
    2641 -        def __str__(self):
    2642                  return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
    2643
    2644  # -----------------------------------------------------------------------
    2645  # Python -> PostgreSQL
    2646  # -----------------------------------------------------------------------
    2647  # test when Squeeze (and thus psycopg2 2.2 becomes Stable
    2648 -class cAdapterPyDateTime(object):
    2649
    2650 -        def __init__(self, dt):
    2651                  if dt.tzinfo is None:
    2652                          raise ValueError('datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat())
    2653                  self.__dt = dt
    2654
    2655 -        def getquoted(self):
    2656                  return _timestamp_template % self.__dt.isoformat()
    2657
    2658  #=======================================================================
    2659  #  main
    2660  #-----------------------------------------------------------------------
    2661
    2662  # make sure psycopg2 knows how to handle unicode ...
    2663  psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
    2664  psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY)
    2665
    2666  # tell psycopg2 how to adapt datetime types with timestamps when locales are in use
    2667  # check in 0.9:
    2668  psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime)
    2669
    2670  # turn dict()s into JSON - only works > 9.2
    2671  #psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)
    2672
    2673  # do NOT adapt *lists* to "... IN (*) ..." syntax because we want
    2674  # them adapted to "... ARRAY[]..." so we can support PG arrays
    2675
    2676  #=======================================================================
    2677  if __name__ == "__main__":
    2678
    2679          if len(sys.argv) < 2:
    2680                  sys.exit()
    2681
    2682          if sys.argv[1] != 'test':
    2683                  sys.exit()
    2684
    2685          from Gnumed.pycommon.gmTools import file2md5
    2686
    2687          logging.basicConfig(level=logging.DEBUG)
    2688
    2689          #--------------------------------------------------------------------
    2690 -        def test_file2bytea():
    2691                  run_rw_queries(queries = [
    2692                          {'cmd': 'drop table if exists test_bytea'},
    2693                          {'cmd': 'create table test_bytea (data bytea)'}
    2694                  ])
    2695
    2696                  try:
    2697                          file2bytea(query = 'insert into test_bytea values (%(data)s::bytea) returning md5(data) as md5', filename = sys.argv[2], file_md5 = file2md5(sys.argv[2], True))
    2698                  except:
    2699                          _log.exception('error')
    2700
    2701                  run_rw_queries(queries = [
    2702                          {'cmd': 'drop table test_bytea'}
    2703                  ])
    2704
    2705          #--------------------------------------------------------------------
    2706 -        def test_file2bytea_lo():
    2707                  lo_oid = file2bytea_lo (
    2708                          filename = sys.argv[2]
    2709                          #, file_md5 = file2md5(sys.argv[2], True)
    2710                  )
    2711                  print(lo_oid)
    2712  #               if lo_oid != -1:
    2713  #                       run_rw_queries(queries = [
    2714  #                               {'cmd': u'select lo_unlink(%(loid)s::oid)', 'args': {'loid': lo_oid}}
    2715  #                       ])
    2716
    2717          #--------------------------------------------------------------------
    2718 -        def test_file2bytea_copy_from():
    2719
    2720                  run_rw_queries(queries = [
    2721                          {'cmd': 'drop table if exists test_bytea'},
    2722                          {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
    2723                          {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
    2724                  ])
    2725
    2726                  md5_query = {
    2727                          'cmd': 'select md5(data) AS md5 FROM test_bytea WHERE pk = %(pk)s',
    2728                          'args': {'pk': 1}
    2729                  }
    2730
    2731                  file2bytea_copy_from (
    2732                          table = 'test_bytea',
    2733                          columns = ['data'],
    2734                          filename = sys.argv[2],
    2735                          md5_query = md5_query,
    2736                          file_md5 = file2md5(sys.argv[2], True)
    2737                  )
    2738
    2739                  run_rw_queries(queries = [
    2740                          {'cmd': 'drop table if exists test_bytea'}
    2741                  ])
    2742
    2743          #--------------------------------------------------------------------
    2744 -        def test_file2bytea_overlay():
    2745
    2746                  run_rw_queries(queries = [
    2747                          {'cmd': 'drop table if exists test_bytea'},
    2748                          {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
    2749                          {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
    2750                  ])
    2751
    2752                  cmd = """
    2753                  update test_bytea
    2754                  set data = overlay (
    2755                          coalesce(data, ''::bytea)
    2756                          placing %(data)s::bytea
    2757                          from %(start)s
    2758                          for %(size)s
    2759                  )
    2760                  where
    2761                          pk > %(pk)s
    2762                  """
    2763                  md5_cmd = 'select md5(data) from test_bytea'
    2764                  args = {'pk': 0}
    2765                  file2bytea_overlay (
    2766                          query = cmd,
    2767                          args = args,
    2768                          filename = sys.argv[2],
    2769                          conn = None,
    2770                          md5_query = md5_cmd,
    2771                          file_md5 = file2md5(sys.argv[2], True)
    2772                  )
    2773
    2774                  run_rw_queries(queries = [
    2775                          {'cmd': 'drop table test_bytea'}
    2776                  ])
    2777
    2778          #--------------------------------------------------------------------
    2779 -        def test_get_connection():
    2780                  print("testing get_connection()")
    2781
    2782                  dsn = 'foo'
    2783                  try:
    2784                          conn = get_connection(dsn=dsn)
    2785                  except dbapi.ProgrammingError as e:
    2786                          print("1) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2787                          t, v = sys.exc_info()[:2]
    2788                          print (' ', t)
    2789                          print (' ', v)
    2790
    2791                  dsn = 'dbname=gnumed_v9'
    2792                  try:
    2793                          conn = get_connection(dsn=dsn)
    2794                  except cAuthenticationError:
    2795                          print("2) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2796                          t, v = sys.exc_info()[:2]
    2797                          print(' ', t)
    2798                          print(' ', v)
    2799
    2800                  dsn = 'dbname=gnumed_v9 user=abc'
    2801                  try:
    2802                          conn = get_connection(dsn=dsn)
    2803                  except cAuthenticationError:
    2804                          print("3) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2805                          t, v = sys.exc_info()[:2]
    2806                          print(' ', t)
    2807                          print(' ', v)
    2808
    2809                  dsn = 'dbname=gnumed_v22 user=any-doc'
    2810                  try:
    2811                          conn = get_connection(dsn=dsn)
    2812                          print("4) SUCCESS:", dsn)
    2813                  except cAuthenticationError:
    2814                          print("4) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2815                          t, v = sys.exc_info()[:2]
    2816                          print(' ', t)
    2817                          print(' ', v)
    2818
    2819                  dsn = 'dbname=gnumed_v9 user=any-doc password=abc'
    2820                  try:
    2821                          conn = get_connection(dsn=dsn)
    2822                  except cAuthenticationError:
    2823                          print("5) SUCCESS: get_connection(%s) failed as expected" % dsn)
    2824                          t, v = sys.exc_info()[:2]
    2825                          print(' ', t)
    2826                          print(' ', v)
    2827
    2828                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2829                  conn = get_connection(dsn=dsn, readonly=True)
    2830
    2831                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2832                  conn = get_connection(dsn=dsn, readonly=False, verbose=True)
    2833
    2834                  dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
    2835                  encoding = 'foo'
    2836                  try:
    2837                          conn = get_connection(dsn=dsn, encoding=encoding)
    2838                  except cEncodingError:
    2839                          print("6) SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding))
    2840                          t, v = sys.exc_info()[:2]
    2841                          print(' ', t)
    2842                          print(' ', v)
    2843
    2844          #--------------------------------------------------------------------
    2845 -        def test_exceptions():
    2846                  print("testing exceptions")
    2847
    2848                  try:
    2849                          raise cAuthenticationError('no dsn', 'no previous exception')
    2850                  except cAuthenticationError:
    2851                          t, v, tb = sys.exc_info()
    2852                          print(t)
    2853                          print(v)
    2854                          print(tb)
    2855
    2856                  try:
    2857                          raise cEncodingError('no dsn', 'no previous exception')
    2858                  except cEncodingError:
    2859                          t, v, tb = sys.exc_info()
    2860                          print(t)
    2861                          print(v)
    2862                          print(tb)
    2863          #--------------------------------------------------------------------
    2864 -        def test_ro_queries():
    2865                  print("testing run_ro_queries()")
    2866
    2867                  dsn = 'dbname=gnumed_v9 user=any-doc password=any-doc'
    2868                  conn = get_connection(dsn, readonly=True)
    2869
    2870                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    2871                  print(data)
    2872                  print(idx)
    2873                  data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True)
    2874                  print(data)
    2875                  print(idx)
    2876
    2877                  curs = conn.cursor()
    2878
    2879                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
    2880                  print(data)
    2881                  print(idx)
    2882
    2883                  data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True, verbose=True)
    2884                  print(data)
    2885                  print(idx)
    2886
    2887                  try:
    2888                          data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'selec 1'}], return_data=True, get_col_idx=True, verbose=True)
    2889                          print(data)
    2890                          print(idx)
    2891                  except psycopg2.ProgrammingError:
    2892                          print('SUCCESS: run_ro_queries("selec 1") failed as expected')
    2893                          t, v = sys.exc_info()[:2]
    2894                          print(' ', t)
    2895                          print(' ', v)
    2896
    2897                  curs.close()
    2898          #--------------------------------------------------------------------
    2899 -        def test_request_dsn():
    2900                  conn = get_connection()
    2901                  print(conn)
    2902                  conn.close()
    2903          #--------------------------------------------------------------------
    2904 -        def test_set_encoding():
    2905                  print("testing set_default_client_encoding()")
    2906
    2907                  enc = 'foo'
    2908                  try:
    2909                          set_default_client_encoding(enc)
    2910                          print("SUCCESS: encoding [%s] worked" % enc)
    2911                  except ValueError:
    2912                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2913                          t, v = sys.exc_info()[:2]
    2914                          print(' ', t)
    2915                          print(' ', v)
    2916
    2917                  enc = ''
    2918                  try:
    2919                          set_default_client_encoding(enc)
    2920                          print("SUCCESS: encoding [%s] worked" % enc)
    2921                  except ValueError:
    2922                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2923                          t, v = sys.exc_info()[:2]
    2924                          print(' ', t)
    2925                          print(' ', v)
    2926
    2927                  enc = 'latin1'
    2928                  try:
    2929                          set_default_client_encoding(enc)
    2930                          print("SUCCESS: encoding [%s] worked" % enc)
    2931                  except ValueError:
    2932                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2933                          t, v = sys.exc_info()[:2]
    2934                          print(' ', t)
    2935                          print(' ', v)
    2936
    2937                  enc = 'utf8'
    2938                  try:
    2939                          set_default_client_encoding(enc)
    2940                          print("SUCCESS: encoding [%s] worked" % enc)
    2941                  except ValueError:
    2942                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2943                          t, v = sys.exc_info()[:2]
    2944                          print(' ', t)
    2945                          print(' ', v)
    2946
    2947                  enc = 'unicode'
    2948                  try:
    2949                          set_default_client_encoding(enc)
    2950                          print("SUCCESS: encoding [%s] worked" % enc)
    2951                  except ValueError:
    2952                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2953                          t, v = sys.exc_info()[:2]
    2954                          print(' ', t)
    2955                          print(' ', v)
    2956
    2957                  enc = 'UNICODE'
    2958                  try:
    2959                          set_default_client_encoding(enc)
    2960                          print("SUCCESS: encoding [%s] worked" % enc)
    2961                  except ValueError:
    2962                          print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
    2963                          t, v = sys.exc_info()[:2]
    2964                          print(' ', t)
    2965                          print(' ', v)
    2966          #--------------------------------------------------------------------
    2967 -        def test_connection_pool():
    2968                  dsn = get_default_dsn()
    2969                  pool = cConnectionPool(minconn=1, maxconn=2, dsn=None, verbose=False)
    2970                  print(pool)
    2971                  print(pool.getconn())
    2972                  print(pool.getconn())
    2973                  print(pool.getconn())
    2974                  print(type(pool.getconn()))
    2975          #--------------------------------------------------------------------
    2976 -        def test_list_args():
    2977                  dsn = get_default_dsn()
    2978                  conn = get_connection(dsn, readonly=True)
    2979                  curs = conn.cursor()
    2980                  curs.execute('SELECT * from clin.clin_narrative where narrative = %s', ['a'])
    2981          #--------------------------------------------------------------------
    2982 -        def test_sanitize_pg_regex():
    2983                  tests = [
    2984                          ['(', '\\(']
    2985                          , ['[', '\\[']
    2986                          , [')', '\\)']
    2987                  ]
    2988                  for test in tests:
    2989                          result = sanitize_pg_regex(test[0])
    2990                          if result != test[1]:
    2991                                  print('ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
    2992          #--------------------------------------------------------------------
    2993 -        def test_is_pg_interval():
    2994                  status = True
    2995                  tests = [
    2996                          [None, True],           # None == NULL == succeeds !
    2997                          [1, True],
    2998                          ['1', True],
    2999                          ['abc', False]
    3000                  ]
    3001
    3002                  if not is_pg_interval():
    3003                          print('ERROR: is_pg_interval() returned "False", expected "True"')
    3004                          status = False
    3005
    3006                  for test in tests:
    3007                          result = is_pg_interval(test[0])
    3008                          if result != test[1]:
    3009                                  print('ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
    3010                                  status = False
    3011
    3012                  return status
    3013          #--------------------------------------------------------------------
    3014 -        def test_sanity_check_time_skew():
    3015                  sanity_check_time_skew()
    3016
    3017          #--------------------------------------------------------------------
    3018 -        def test_get_foreign_key_names():
    3019                  print(get_foreign_key_names (
    3020                          src_schema = 'clin',
    3021                          src_table = 'vaccination',
    3022                          src_column = 'fk_episode',
    3023                          target_schema = 'clin',
    3024                          target_table = 'episode',
    3025                          target_column = 'pk'
    3026                  ))
    3027
    3028          #--------------------------------------------------------------------
    3029 -        def test_get_foreign_key_details():
    3030                  schema = 'clin'
    3031                  table = 'episode'
    3032                  col = 'pk'
    3033                  print('column %s.%s.%s is referenced by:' % (schema, table, col))
    3034                  for row in get_foreign_keys2column (
    3035                          schema = schema,
    3036                          table = table,
    3037                          column = col
    3038                  ):
    3039                          print(' <- %s.%s' % (
    3040                                  row['referencing_table'],
    3041                                  row['referencing_column']
    3042                          ))
    3043
    3044          #--------------------------------------------------------------------
    3045 -        def test_set_user_language():
    3046                  # (user, language, result, exception type)
    3047                  tests = [
    3048                          # current user
    3049                          [None, 'de_DE', True],
    3050                          [None, 'lang_w/o_tx', False],
    3051                          [None, None, True],
    3052                          # valid user
    3053                          ['any-doc', 'de_DE', True],
    3054                          ['any-doc', 'lang_w/o_tx', False],
    3055                          ['any-doc', None, True],
    3056                          # invalid user
    3057                          ['invalid user', 'de_DE', None],
    3058                          ['invalid user', 'lang_w/o_tx', False], # lang checking happens before user checking
    3059                          ['invalid user', None, True]
    3060                  ]
    3061                  for test in tests:
    3062                          try:
    3063                                  result = set_user_language(user = test[0], language = test[1])
    3064                                  if result != test[2]:
    3065                                          print("test:", test)
    3066                                          print("result:", result, "expected:", test[2])
    3067                          except psycopg2.IntegrityError as e:
    3068                                  if test[2] is None:
    3069                                          continue
    3070                                  print("test:", test)
    3071                                  print("expected exception")
    3072                                  print("result:", e)
    3073
    3074          #--------------------------------------------------------------------
    3075 -        def test_get_schema_revision_history():
    3076                  for line in get_schema_revision_history():
    3077                          print(' - '.join(line))
    3078
    3079          #--------------------------------------------------------------------
    3080 -        def test_run_query():
    3081                  gmDateTime.init()
    3082                  args = {'dt': gmDateTime.pydt_max_here()}
    3083                  cmd = "SELECT %(dt)s"
    3084
    3085                  #cmd = u"SELECT 'infinity'::timestamp with time zone"
    3086
    3087                  cmd = """
    3088  SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
    3089          SELECT REGEXP_REPLACE (
    3090                  't1.130729.0902.tif',                   -- string
    3091                  E'(.1)\.([0-9\.]+)(\.tif)',             -- pattern
    3092                  E'\\\\2'                                                -- replacement
    3093          ) AS foofoo
    3094  ) AS foo"""
    3095                  rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
    3096                  print(rows)
    3097                  print(rows[0])
    3098                  print(rows[0][0])
    3099          #--------------------------------------------------------------------
    3100 -        def test_schema_exists():
    3101                  print(schema_exists())
    3102          #--------------------------------------------------------------------
    3103 -        def test_row_locks():
    3104                  row_is_locked(table = 'dem.identity', pk = 12)
    3105
    3106                  print("1st connection:")
    3107                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3108                  print(" 1st shared lock succeeded:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3109                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3110
    3111                  print("   2nd shared lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3112                  print("   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3113                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3114                  print("   exclusive lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = True))
    3115                  print("   `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = True))
    3116                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3117
    3118                  print("2nd connection:")
    3119                  conn = get_raw_connection(readonly=True)
    3120                  print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3121                  print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3122                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3123                  print(" exclusive lock succeeded ?", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True), "(should fail)")
    3124                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3125
    3126                  print("1st connection:")
    3127                  print(" unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
    3128                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3129
    3130                  print("2nd connection:")
    3131                  print(" exclusive lock should succeed", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True))
    3132                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3133                  print("  shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3134                  print("  `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3135                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3136                  print(" unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
    3137                  print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
    3138
    3139                  conn.close()
    3140
    3141          #--------------------------------------------------------------------
    3142 -        def test_get_foreign_key_names():
    3143                  print(get_foreign_key_names (
    3144                          src_schema = 'dem',
    3145                          src_table = 'names',
    3146                          src_column = 'id_identity',
    3147                          target_schema = 'dem',
    3148                          target_table = 'identity',
    3149                          target_column = 'pk'
    3150                  ))
    3151
    3152          #--------------------------------------------------------------------
    3153 -        def test_get_index_name():
    3154                  print(get_index_name(indexed_table = 'clin.vaccination', indexed_column = 'fk_episode'))
    3155
    3156          #--------------------------------------------------------------------
    3157 -        def test_faulty_SQL():
    3158                  run_rw_queries(queries = [{'cmd': 'SELEC 1'}])
    3159
    3160          #--------------------------------------------------------------------
    3161 -        def test_log_settings():
    3162                  conn = conn = get_connection()
    3163                  _log_PG_settings(curs = conn.cursor())
    3164
    3165          #--------------------------------------------------------------------
    3166          # run tests
    3167          test_get_connection()
    3168          #test_exceptions()
    3169          #test_ro_queries()
    3170          #test_request_dsn()
    3171          #test_set_encoding()
    3172          #test_connection_pool()
    3173          #test_list_args()
    3174          #test_sanitize_pg_regex()
    3175          #test_is_pg_interval()
    3176          #test_sanity_check_time_skew()
    3177          #test_get_foreign_key_details()
    3178          #test_get_foreign_key_names()
    3179          #test_get_index_name()
    3180          #test_set_user_language()
    3181          #test_get_schema_revision_history()
    3182          #test_run_query()
    3183          #test_schema_exists()
    3184          #test_get_foreign_key_names()
    3185          #test_row_locks()
    3186          #test_file2bytea()
    3187          #test_file2bytea_overlay()
    3188          #test_file2bytea_copy_from()
    3189          #test_file2bytea_lo()
    3190          #test_faulty_SQL()
    3191          #test_log_settings()
    3192
    3193  # ======================================================================
    3194