Gnumed.pycommon.gmTools

Package Gnumed :: Package pycommon :: Module gmTools |
[frames] | no frames]

# Source Code for Module Gnumed.pycommon.gmTools

       1  # -*- coding: utf-8 -*-
       2
       3
       4  __doc__ = """GNUmed general tools."""
       5
       6  #===========================================================================
       7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>"
       8  __license__ = "GPL v2 or later (details at http://www.gnu.org)"
       9
      10  # std libs
      11  import sys
      12  import os
      13  import os.path
      14  import csv
      15  import tempfile
      16  import logging
      17  import hashlib
      18  import platform
      19  import subprocess
      20  import decimal
      21  import getpass
      22  import io
      23  import functools
      24  import json
      25  import shutil
      26  import zipfile
      27  import datetime as pydt
      28  import re as regex
      29  import xml.sax.saxutils as xml_tools
      30  # old:
      31  import pickle, zlib
      32  # docutils
      33  du_core = None
      34
      35
      36  # GNUmed libs
      37  if __name__ == '__main__':
      38          sys.path.insert(0, '../../')
      39  from Gnumed.pycommon import gmBorg
      40
      41
      42  _log = logging.getLogger('gm.tools')
      43
      44  # CAPitalization modes:
      45  (       CAPS_NONE,                                      # don't touch it
      46          CAPS_FIRST,                                     # CAP first char, leave rest as is
      47          CAPS_ALLCAPS,                           # CAP all chars
      48          CAPS_WORDS,                                     # CAP first char of every word
      49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart)
      50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest
      51  ) = range(6)
      52
      53
      54  u_currency_pound = '\u00A3'                             # Pound sign
      55  u_currency_sign = '\u00A4'                                      # generic currency sign
      56  u_currency_yen = '\u00A5'                                       # Yen sign
      57  u_right_double_angle_quote = '\u00AB'           # <<
      58  u_registered_trademark = '\u00AE'
      59  u_plus_minus = '\u00B1'
      60  u_superscript_one = '\u00B9'                            # ^1
      61  u_left_double_angle_quote = '\u00BB'            # >>
      62  u_one_quarter = '\u00BC'
      63  u_one_half = '\u00BD'
      64  u_three_quarters = '\u00BE'
      65  u_multiply = '\u00D7'                                           # x
      66  u_greek_ALPHA = '\u0391'
      67  u_greek_alpha = '\u03b1'
      68  u_greek_OMEGA = '\u03A9'
      69  u_greek_omega = '\u03c9'
      70  u_dagger = '\u2020'
      71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>)
      72  u_ellipsis = '\u2026'                                                   # ...
      73  u_euro = '\u20AC'                                                               # EURO sign
      74  u_numero = '\u2116'                                                             # No. / # sign
      75  u_down_left_arrow = '\u21B5'                                    # <-'
      76  u_left_arrow = '\u2190'                                                 # <--
      77  u_up_arrow = '\u2191'
      78  u_arrow2right = '\u2192'                                                # -->
      79  u_down_arrow = '\u2193'
      80  u_left_arrow_with_tail = '\u21a2'                               # <--<
      81  u_arrow2right_from_bar = '\u21a6'                               # |->
      82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->|
      83  u_sum = '\u2211'                                                                # sigma
      84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly
      85  u_corresponds_to = '\u2258'
      86  u_infinity = '\u221E'
      87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->|
      88  u_diameter = '\u2300'
      89  u_checkmark_crossed_out = '\u237B'
      90  u_box_vert_left = '\u23b8'
      91  u_box_vert_right = '\u23b9'
      92  u_box_horiz_single = '\u2500'                           # -
      93  u_box_vert_light = '\u2502'
      94  u_box_horiz_light_3dashes = '\u2504'            # ...
      95  u_box_vert_light_4dashes = '\u2506'
      96  u_box_horiz_4dashes = '\u2508'                          # ....
      97  u_box_T_right = '\u251c'                                        # |-
      98  u_box_T_left = '\u2524'                                         # -|
      99  u_box_T_down = '\u252c'
     100  u_box_T_up = '\u2534'
     101  u_box_plus = '\u253c'
     102  u_box_top_double = '\u2550'
     103  u_box_top_left_double_single = '\u2552'
     104  u_box_top_right_double_single = '\u2555'
     105  u_box_top_left_arc = '\u256d'
     106  u_box_top_right_arc = '\u256e'
     107  u_box_bottom_right_arc = '\u256f'
     108  u_box_bottom_left_arc = '\u2570'
     109  u_box_horiz_light_heavy = '\u257c'
     110  u_box_horiz_heavy_light = '\u257e'
     111  u_skull_and_crossbones = '\u2620'
     112  u_caduceus = '\u2624'
     113  u_frowning_face = '\u2639'
     114  u_smiling_face = '\u263a'
     115  u_black_heart = '\u2665'
     116  u_female = '\u2640'
     117  u_male = '\u2642'
     118  u_male_female = '\u26a5'
     119  u_checkmark_thin = '\u2713'
     120  u_checkmark_thick = '\u2714'
     121  u_heavy_greek_cross = '\u271a'
     122  u_arrow2right_thick = '\u2794'
     123  u_writing_hand = '\u270d'
     124  u_pencil_1 = '\u270e'
     125  u_pencil_2 = '\u270f'
     126  u_pencil_3 = '\u2710'
     127  u_latin_cross = '\u271d'
     128  u_arrow2right_until_black_diamond = '\u291e'    # ->*
     129  u_kanji_yen = '\u5186'                                                  # Yen kanji
     130  u_replacement_character = '\ufffd'
     131  u_link_symbol = '\u1f517'
     132
     133  _kB = 1024
     134  _MB = 1024 * _kB
     135  _GB = 1024 * _MB
     136  _TB = 1024 * _GB
     137  _PB = 1024 * _TB
     138
     139  #===========================================================================
     140 -def handle_uncaught_exception_console(t, v, tb):
     141
     142          print(".========================================================")
     143          print("| Unhandled exception caught !")
     144          print("| Type :", t)
     145          print("| Value:", v)
     146          print("`========================================================")
     147          _log.critical('unhandled exception caught', exc_info = (t,v,tb))
     148          sys.__excepthook__(t,v,tb)
     149
     150  #===========================================================================
     151  # path level operations
     152  #---------------------------------------------------------------------------
     153 -def mkdir(directory=None, mode=None):
     154          try:
     155                  if mode is None:
     156                          os.makedirs(directory)
     157                  else:
     158                          old_umask = os.umask(0)
     159                          os.makedirs(directory, mode)
     160                          os.umask(old_umask)
     161          except OSError as e:
     162                  if (e.errno == 17) and not os.path.isdir(directory):
     163                          raise
     164          return True
     165
     166  #---------------------------------------------------------------------------
     167 -def rmdir(directory):
     168          #-------------------------------
     169          def _on_rm_error(func, path, exc):
     170                  _log.error('error while shutil.rmtree(%s)', path, exc_info=exc)
     171                  return True
     172          #-------------------------------
     173          error_count = 0
     174          try:
     175                  shutil.rmtree(directory, False, _on_rm_error)
     176          except Exception:
     177                  _log.exception('cannot shutil.rmtree(%s)', directory)
     178                  error_count += 1
     179          return error_count
     180
     181  #---------------------------------------------------------------------------
     182 -def rm_dir_content(directory):
     183          _log.debug('cleaning out [%s]', directory)
     184          try:
     185                  items = os.listdir(directory)
     186          except OSError:
     187                  return False
     188          for item in items:
     189                  # attempt file/link removal and ignore (but log) errors
     190                  full_item = os.path.join(directory, item)
     191                  try:
     192                          os.remove(full_item)
     193                  except OSError:         # as per the docs, this is a directory
     194                          _log.debug('[%s] seems to be a subdirectory', full_item)
     195                          errors = rmdir(full_item)
     196                          if errors > 0:
     197                                  return False
     198                  except Exception:
     199                          _log.exception('cannot os.remove(%s) [a file or a link]', full_item)
     200                          return False
     201
     202          return True
     203
     204  #---------------------------------------------------------------------------
     205 -def mk_sandbox_dir(prefix=None, base_dir=None):
     206          if base_dir is None:
     207                  base_dir = gmPaths().tmp_dir
     208          if prefix is None:
     209                  prefix = 'sandbox-'
     210          return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
     211
     212  #---------------------------------------------------------------------------
     213 -def parent_dir(directory):
     214          return os.path.abspath(os.path.join(directory, '..'))
     215
     216  #---------------------------------------------------------------------------
     217 -def dirname_stem(directory):
     218          # /home/user/dir/ -> dir
     219          # /home/user/dir  -> dir
     220          return os.path.basename(os.path.normpath(directory))            # normpath removes trailing slashes if any
     221
     222  #---------------------------------------------------------------------------
     223 -def dir_is_empty(directory=None):
     224          try:
     225                  return len(os.listdir(directory)) == 0
     226          except OSError as exc:
     227                  if exc.errno == 2:
     228                          return None
     229                  raise
     230
     231  #---------------------------------------------------------------------------
     232 -class gmPaths(gmBorg.cBorg):
     233          """This class provides the following paths:
     234
     235          .home_dir                               user home
     236          .local_base_dir                 script installation dir
     237          .working_dir                    current dir
     238          .user_config_dir
     239          .system_config_dir
     240          .system_app_data_dir    (not writable)
     241          .tmp_dir                                instance-local
     242          .user_tmp_dir                   user-local (NOT per instance)
     243          .bytea_cache_dir                caches downloaded BYTEA data
     244          """
     245 -        def __init__(self, app_name=None, wx=None):
     246                  """Setup pathes.
     247
     248                  <app_name> will default to (name of the script - .py)
     249                  """
     250                  try:
     251                          self.already_inited
     252                          return
     253                  except AttributeError:
     254                          pass
     255
     256                  self.init_paths(app_name=app_name, wx=wx)
     257                  self.already_inited = True
     258
     259          #--------------------------------------
     260          # public API
     261          #--------------------------------------
     262 -        def init_paths(self, app_name=None, wx=None):
     263
     264                  if wx is None:
     265                          _log.debug('wxPython not available')
     266                  _log.debug('detecting paths directly')
     267
     268                  if app_name is None:
     269                          app_name, ext = os.path.splitext(os.path.basename(sys.argv[0]))
     270                          _log.info('app name detected as [%s]', app_name)
     271                  else:
     272                          _log.info('app name passed in as [%s]', app_name)
     273
     274                  # the user home, doesn't work in Wine so work around that
     275                  self.__home_dir = None
     276
     277                  # where the main script (the "binary") is installed
     278                  if getattr(sys, 'frozen', False):
     279                          _log.info('frozen app, installed into temporary path')
     280                          # this would find the path of *THIS* file
     281                          #self.local_base_dir = os.path.dirname(__file__)
     282                          # while this is documented on the web, the ${_MEIPASS2} does not exist
     283                          #self.local_base_dir = os.environ.get('_MEIPASS2')
     284                          # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use
     285                          # when asking about this on pyinstaller@googlegroups.com
     286                          #self.local_base_dir = sys._MEIPASS
     287                          # however, we are --onedir, so we should look at sys.executable
     288                          # as per the pyinstaller manual
     289                          self.local_base_dir = os.path.dirname(sys.executable)
     290                  else:
     291                          self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
     292
     293                  # the current working dir at the OS
     294                  self.working_dir = os.path.abspath(os.curdir)
     295
     296                  # user-specific config dir, usually below the home dir
     297                  mkdir(os.path.join(self.home_dir, '.%s' % app_name))
     298                  self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name)
     299
     300                  # system-wide config dir, usually below /etc/ under UN*X
     301                  try:
     302                          self.system_config_dir = os.path.join('/etc', app_name)
     303                  except ValueError:
     304                          #self.system_config_dir = self.local_base_dir
     305                          self.system_config_dir = self.user_config_dir
     306
     307                  # system-wide application data dir
     308                  try:
     309                          self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name)
     310                  except ValueError:
     311                          self.system_app_data_dir = self.local_base_dir
     312
     313                  # temporary directory
     314                  try:
     315                          self.__tmp_dir_already_set
     316                          _log.debug('temp dir already set')
     317                  except AttributeError:
     318                          _log.info('temp file prefix: %s', tempfile.gettempprefix())
     319                          _log.info('initial (user level) temp dir: %s', tempfile.gettempdir())
     320                          # $TMP/gnumed-$USER/
     321                          self.user_tmp_dir = os.path.join(tempfile.gettempdir(), app_name + '-' + getpass.getuser())
     322                          mkdir(self.user_tmp_dir, 0o700)
     323                          tempfile.tempdir = self.user_tmp_dir
     324                          _log.info('intermediate (app level) temp dir: %s', tempfile.gettempdir())
     325                          # $TMP/gnumed-$USER/g$UNIQUE/
     326                          self.tmp_dir = tempfile.mkdtemp(prefix = 'g-')
     327                          _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir())
     328
     329                  # BYTEA cache dir
     330                  cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache')
     331                  try:
     332                          stat = os.stat(cache_dir)
     333                          _log.warning('reusing BYTEA cache dir: %s', cache_dir)
     334                          _log.debug(stat)
     335                  except FileNotFoundError:
     336                          mkdir(cache_dir, mode = 0o0700)
     337                  self.bytea_cache_dir = cache_dir
     338
     339                  self.__log_paths()
     340                  if wx is None:
     341                          return True
     342
     343                  # retry with wxPython
     344                  _log.debug('re-detecting paths with wxPython')
     345
     346                  std_paths = wx.StandardPaths.Get()
     347                  _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName())
     348
     349                  # user-specific config dir, usually below the home dir
     350                  mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name))
     351                  self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)
     352
     353                  # system-wide config dir, usually below /etc/ under UN*X
     354                  try:
     355                          tmp = std_paths.GetConfigDir()
     356                          if not tmp.endswith(app_name):
     357                                  tmp = os.path.join(tmp, app_name)
     358                          self.system_config_dir = tmp
     359                  except ValueError:
     360                          # leave it at what it was from direct detection
     361                          pass
     362
     363                  # system-wide application data dir
     364                  # Robin attests that the following doesn't always
     365                  # give sane values on Windows, so IFDEF it
     366                  if 'wxMSW' in wx.PlatformInfo:
     367                          _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir')
     368                  else:
     369                          try:
     370                                  self.system_app_data_dir = std_paths.GetDataDir()
     371                          except ValueError:
     372                                  pass
     373
     374                  self.__log_paths()
     375                  return True
     376
     377          #--------------------------------------
     378 -        def __log_paths(self):
     379                  _log.debug('sys.argv[0]: %s', sys.argv[0])
     380                  _log.debug('sys.executable: %s', sys.executable)
     381                  _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>'))
     382                  _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>'))
     383                  _log.debug('__file__ : %s', __file__)
     384                  _log.debug('local application base dir: %s', self.local_base_dir)
     385                  _log.debug('current working dir: %s', self.working_dir)
     386                  _log.debug('user home dir: %s', self.home_dir)
     387                  _log.debug('user-specific config dir: %s', self.user_config_dir)
     388                  _log.debug('system-wide config dir: %s', self.system_config_dir)
     389                  _log.debug('system-wide application data dir: %s', self.system_app_data_dir)
     390                  _log.debug('temporary dir (user): %s', self.user_tmp_dir)
     391                  _log.debug('temporary dir (instance): %s', self.tmp_dir)
     392                  _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
     393
     394          #--------------------------------------
     395          # properties
     396          #--------------------------------------
     397 -        def _set_user_config_dir(self, path):
     398                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     399                          msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     400                          _log.error(msg)
     401                          raise ValueError(msg)
     402                  self.__user_config_dir = path
     403
     404 -        def _get_user_config_dir(self):
     405                  return self.__user_config_dir
     406
     407          user_config_dir = property(_get_user_config_dir, _set_user_config_dir)
     408          #--------------------------------------
     409 -        def _set_system_config_dir(self, path):
     410                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     411                          msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     412                          _log.error(msg)
     413                          raise ValueError(msg)
     414                  self.__system_config_dir = path
     415
     416 -        def _get_system_config_dir(self):
     417                  return self.__system_config_dir
     418
     419          system_config_dir = property(_get_system_config_dir, _set_system_config_dir)
     420          #--------------------------------------
     421 -        def _set_system_app_data_dir(self, path):
     422                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     423                          msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     424                          _log.error(msg)
     425                          raise ValueError(msg)
     426                  self.__system_app_data_dir = path
     427
     428 -        def _get_system_app_data_dir(self):
     429                  return self.__system_app_data_dir
     430
     431          system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir)
     432          #--------------------------------------
     433 -        def _set_home_dir(self, path):
     434                  raise ValueError('invalid to set home dir')
     435
     436 -        def _get_home_dir(self):
     437                  if self.__home_dir is not None:
     438                          return self.__home_dir
     439
     440                  tmp = os.path.expanduser('~')
     441                  if tmp == '~':
     442                          _log.error('this platform does not expand ~ properly')
     443                          try:
     444                                  tmp = os.environ['USERPROFILE']
     445                          except KeyError:
     446                                  _log.error('cannot access $USERPROFILE in environment')
     447
     448                  if not (
     449                          os.access(tmp, os.R_OK)
     450                                  and
     451                          os.access(tmp, os.X_OK)
     452                                  and
     453                          os.access(tmp, os.W_OK)
     454                  ):
     455                          msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp)
     456                          _log.error(msg)
     457                          raise ValueError(msg)
     458
     459                  self.__home_dir = tmp
     460                  return self.__home_dir
     461
     462          home_dir = property(_get_home_dir, _set_home_dir)
     463
     464          #--------------------------------------
     465 -        def _set_tmp_dir(self, path):
     466                  if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)):
     467                          msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path)
     468                          _log.error(msg)
     469                          raise ValueError(msg)
     470                  _log.debug('previous temp dir: %s', tempfile.gettempdir())
     471                  self.__tmp_dir = path
     472                  tempfile.tempdir = self.__tmp_dir
     473                  _log.debug('new temp dir: %s', tempfile.gettempdir())
     474                  self.__tmp_dir_already_set = True
     475
     476 -        def _get_tmp_dir(self):
     477                  return self.__tmp_dir
     478
     479          tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
     480
     481  #===========================================================================
     482  # file related tools
     483  #---------------------------------------------------------------------------
     484 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
     485          if target_encoding is None:
     486                  return source_file
     487          if target_encoding == source_encoding:
     488                  return source_file
     489          if target_file is None:
     490                  target_file = get_unique_filename (
     491                          prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding),
     492                          suffix = fname_extension(source_file, '.txt'),
     493                          tmp_dir = base_dir
     494                  )
     495
     496          _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file)
     497
     498          in_file = io.open(source_file, mode = 'rt', encoding = source_encoding)
     499          out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode)
     500          for line in in_file:
     501                  out_file.write(line)
     502          out_file.close()
     503          in_file.close()
     504
     505          return target_file
     506
     507  #---------------------------------------------------------------------------
     508 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
     509          _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir)
     510          success = False
     511          try:
     512                  with zipfile.ZipFile(archive_name) as archive:
     513                          archive.extractall(target_dir)
     514                  success = True
     515          except Exception:
     516                  _log.exception('cannot unzip')
     517                  return False
     518          if remove_archive:
     519                  remove_file(archive_name)
     520          return success
     521
     522  #---------------------------------------------------------------------------
     523 -def remove_file(filename, log_error=True, force=False):
     524          if not os.path.lexists(filename):
     525                  return True
     526
     527          # attempt file remove and ignore (but log) errors
     528          try:
     529                  os.remove(filename)
     530                  return True
     531
     532          except Exception:
     533                  if log_error:
     534                          _log.exception('cannot os.remove(%s)', filename)
     535
     536          if force:
     537                  tmp_name = get_unique_filename(tmp_dir = fname_dir(filename))
     538                  _log.debug('attempting os.replace() to: %s', tmp_name)
     539                  try:
     540                          os.replace(filename, tmp_name)
     541                          return True
     542
     543                  except BaseException:
     544                          if log_error:
     545                                  _log.exception('cannot os.remove(%s)', filename)
     546
     547          return False
     548
     549  #---------------------------------------------------------------------------
     550 -def file2md5(filename=None, return_hex=True):
     551          blocksize = 2**10 * 128                 # 128k, since md5 uses 128 byte blocks
     552          _log.debug('md5(%s): <%s> byte blocks', filename, blocksize)
     553
     554          f = io.open(filename, mode = 'rb')
     555
     556          md5 = hashlib.md5()
     557          while True:
     558                  data = f.read(blocksize)
     559                  if not data:
     560                          break
     561                  md5.update(data)
     562          f.close()
     563
     564          _log.debug('md5(%s): %s', filename, md5.hexdigest())
     565
     566          if return_hex:
     567                  return md5.hexdigest()
     568          return md5.digest()
     569
     570  #---------------------------------------------------------------------------
     571 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
     572          _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size)
     573          md5_concat = ''
     574          f = open(filename, 'rb')
     575          while True:
     576                  md5 = hashlib.md5()
     577                  data = f.read(chunk_size)
     578                  if not data:
     579                          break
     580                  md5.update(data)
     581                  md5_concat += md5.hexdigest()
     582          f.close()
     583          md5 = hashlib.md5()
     584          md5.update(md5_concat)
     585          hex_digest = md5.hexdigest()
     586          _log.debug('md5("%s"): %s', md5_concat, hex_digest)
     587          return hex_digest
     588
     589  #---------------------------------------------------------------------------
     590  default_csv_reader_rest_key = 'list_of_values_of_unknown_fields'
     591
     592 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
     593          try:
     594                  is_dict_reader = kwargs['dict']
     595                  del kwargs['dict']
     596          except KeyError:
     597                  is_dict_reader = False
     598
     599          if is_dict_reader:
     600                  kwargs['restkey'] = default_csv_reader_rest_key
     601                  return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs)
     602          return csv.reader(csv_data, dialect=dialect, **kwargs)
     603
     604
     605
     606
     607 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
     608          for line in unicode_csv_data:
     609                  yield line.encode(encoding)
     610
     611  #def utf_8_encoder(unicode_csv_data):
     612  #       for line in unicode_csv_data:
     613  #               yield line.encode('utf-8')
     614
     615 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
     616
     617          # csv.py doesn't do Unicode; encode temporarily as UTF-8:
     618          try:
     619                  is_dict_reader = kwargs['dict']
     620                  del kwargs['dict']
     621                  if is_dict_reader is not True:
     622                          raise KeyError
     623                  kwargs['restkey'] = default_csv_reader_rest_key
     624                  csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
     625          except KeyError:
     626                  is_dict_reader = False
     627                  csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs)
     628
     629          for row in csv_reader:
     630                  # decode ENCODING back to Unicode, cell by cell:
     631                  if is_dict_reader:
     632                          for key in row.keys():
     633                                  if key == default_csv_reader_rest_key:
     634                                          old_data = row[key]
     635                                          new_data = []
     636                                          for val in old_data:
     637                                                  new_data.append(str(val, encoding))
     638                                          row[key] = new_data
     639                                          if default_csv_reader_rest_key not in csv_reader.fieldnames:
     640                                                  csv_reader.fieldnames.append(default_csv_reader_rest_key)
     641                                  else:
     642                                          row[key] = str(row[key], encoding)
     643                          yield row
     644                  else:
     645                          yield [ str(cell, encoding) for cell in row ]
     646                          #yield [str(cell, 'utf-8') for cell in row]
     647
     648  #---------------------------------------------------------------------------
     649 -def fname_sanitize(filename):
     650          """Normalizes unicode, removes non-alpha characters, converts spaces to underscores."""
     651
     652          dir_part, name_part = os.path.split(filename)
     653          if name_part == '':
     654                  return filename
     655
     656          import unicodedata
     657          name_part = unicodedata.normalize('NFKD', name_part)
     658          # remove everything not in group []
     659          name_part = regex.sub (
     660                  '[^.\w\s[\]()%§+-]',
     661                  '',
     662                  name_part,
     663                  flags = regex.UNICODE
     664          ).strip()
     665          # translate whitespace to underscore
     666          name_part = regex.sub (
     667                  '\s+',
     668                  '_',
     669                  name_part,
     670                  flags = regex.UNICODE
     671          )
     672          return os.path.join(dir_part, name_part)
     673
     674  #---------------------------------------------------------------------------
     675 -def fname_stem(filename):
     676          """/home/user/dir/filename.ext -> filename"""
     677          return os.path.splitext(os.path.basename(filename))[0]
     678
     679  #---------------------------------------------------------------------------
     680 -def fname_stem_with_path(filename):
     681          """/home/user/dir/filename.ext -> /home/user/dir/filename"""
     682          return os.path.splitext(filename)[0]
     683
     684  #---------------------------------------------------------------------------
     685 -def fname_extension(filename=None, fallback=None):
     686          """     /home/user/dir/filename.ext -> .ext
     687                  '' or '.' -> fallback if any else ''
     688          """
     689          ext = os.path.splitext(filename)[1]
     690          if ext.strip() not in ['.', '']:
     691                  return ext
     692          if fallback is None:
     693                  return ''
     694          return fallback
     695
     696  #---------------------------------------------------------------------------
     697 -def fname_dir(filename):
     698          # /home/user/dir/filename.ext -> /home/user/dir
     699          return os.path.split(filename)[0]
     700
     701  #---------------------------------------------------------------------------
     702 -def fname_from_path(filename):
     703          # /home/user/dir/filename.ext -> filename.ext
     704          return os.path.split(filename)[1]
     705
     706  #---------------------------------------------------------------------------
     707 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
     708          """This function has a race condition between
     709                          its file.close()
     710             and actually
     711                          using the filename in callers.
     712
     713          The file will NOT exist after calling this function.
     714          """
     715          if tmp_dir is not None:
     716                  if (
     717                          not os.access(tmp_dir, os.F_OK)
     718                                  or
     719                          not os.access(tmp_dir, os.X_OK | os.W_OK)
     720                  ):
     721                          _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir)
     722                          tmp_dir = None
     723
     724          if include_timestamp:
     725                  ts = pydt.datetime.now().strftime('%m%d-%H%M%S-')
     726          else:
     727                  ts = ''
     728
     729          kwargs = {
     730                  'dir': tmp_dir,
     731                  #  make sure file gets deleted as soon as
     732                  # .close()d so we can "safely" open it again
     733                  'delete': True
     734          }
     735
     736          if prefix is None:
     737                  kwargs['prefix'] = 'gm-%s' % ts
     738          else:
     739                  kwargs['prefix'] = prefix + ts
     740
     741          if suffix in [None, '']:
     742                  kwargs['suffix'] = '.tmp'
     743          else:
     744                  if not suffix.startswith('.'):
     745                          suffix = '.' + suffix
     746                  kwargs['suffix'] = suffix
     747
     748          f = tempfile.NamedTemporaryFile(**kwargs)
     749          filename = f.name
     750          f.close()
     751
     752          return filename
     753
     754  #---------------------------------------------------------------------------
     755 -def __make_symlink_on_windows(physical_name, link_name):
     756          import ctypes
     757          #windows_create_symlink = ctypes.windll.kernel32.CreateSymbolicLinkW
     758          kernel32 = ctype.WinDLL('kernel32', use_last_error = True)
     759          windows_create_symlink = kernel32.CreateSymbolicLinkW
     760          windows_create_symlink.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32)
     761          windows_create_symlink.restype = ctypes.c_ubyte
     762          if os.path.isdir(physical_name):
     763                  flags = 1
     764          else:
     765                  flags = 0
     766          ret_code = windows_create_symlink(link_name, physical_name.replace('/', '\\'), flags)
     767          _log.debug('ctypes.windll.kernel32.CreateSymbolicLinkW() [%s] exit code: %s', windows_create_symlink, ret_code)
     768          if ret_code == 0:
     769                  raise ctypes.WinError()
     770          return ret_code
     771
     772  #---------------------------------------------------------------------------
     773 -def mklink(physical_name, link_name, overwrite=False):
     774
     775          _log.debug('creating symlink (overwrite = %s):', overwrite)
     776          _log.debug('link [%s] =>', link_name)
     777          _log.debug('=> physical [%s]', physical_name)
     778
     779          if os.path.exists(link_name):
     780                  _log.debug('link exists')
     781                  if overwrite:
     782                          return True
     783                  return False
     784
     785          try:
     786                  os.symlink(physical_name, link_name)
     787          except (AttributeError, NotImplementedError):
     788                  _log.debug('this Python does not have os.symlink(), trying via ctypes')
     789                  __make_symlink_on_windows(physical_name, link_name)
     790          except PermissionError:
     791                  _log.exception('cannot create link')
     792                  return False
     793          #except OSError:
     794          #       unpriviledged on Windows
     795          return True
     796
     797  #===========================================================================
     798 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
     799          """Import a module from any location."""
     800
     801          _log.debug('CWD: %s', os.getcwd())
     802
     803          remove_path = always_remove_path or False
     804          if module_path not in sys.path:
     805                  _log.info('appending to sys.path: [%s]' % module_path)
     806                  sys.path.append(module_path)
     807                  remove_path = True
     808
     809          _log.debug('will remove import path: %s', remove_path)
     810
     811          if module_name.endswith('.py'):
     812                  module_name = module_name[:-3]
     813
     814          try:
     815                  module = __import__(module_name)
     816          except Exception:
     817                  _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path))
     818                  while module_path in sys.path:
     819                          sys.path.remove(module_path)
     820                  raise
     821
     822          _log.info('imported module [%s] as [%s]' % (module_name, module))
     823          if remove_path:
     824                  while module_path in sys.path:
     825                          sys.path.remove(module_path)
     826
     827          return module
     828
     829  #===========================================================================
     830  # text related tools
     831  #---------------------------------------------------------------------------
     832 -def size2str(size=0, template='%s'):
     833          if size == 1:
     834                  return template % _('1 Byte')
     835          if size < 10 * _kB:
     836                  return template % _('%s Bytes') % size
     837          if size < _MB:
     838                  return template % '%.1f kB' % (float(size) / _kB)
     839          if size < _GB:
     840                  return template % '%.1f MB' % (float(size) / _MB)
     841          if size < _TB:
     842                  return template % '%.1f GB' % (float(size) / _GB)
     843          if size < _PB:
     844                  return template % '%.1f TB' % (float(size) / _TB)
     845          return template % '%.1f PB' % (float(size) / _PB)
     846
     847  #---------------------------------------------------------------------------
     848 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
     849          if boolean is None:
     850                  return none_return
     851          if boolean:
     852                  return true_return
     853          if not boolean:
     854                  return false_return
     855          raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
     856
     857  #---------------------------------------------------------------------------
     858 -def bool2str(boolean=None, true_str='True', false_str='False'):
     859          return bool2subst (
     860                  boolean = bool(boolean),
     861                  true_return = true_str,
     862                  false_return = false_str
     863          )
     864
     865  #---------------------------------------------------------------------------
     866 -def none_if(value=None, none_equivalent=None, strip_string=False):
     867          """Modelled after the SQL NULLIF function."""
     868          if value is None:
     869                  return None
     870          if strip_string:
     871                  stripped = value.strip()
     872          else:
     873                  stripped = value
     874          if stripped == none_equivalent:
     875                  return None
     876          return value
     877
     878  #---------------------------------------------------------------------------
     879 -def coalesce(initial=None, instead=None, template_initial=None, template_instead=None, none_equivalents=None, function_initial=None):
     880          """Modelled after the SQL coalesce function.
     881
     882          To be used to simplify constructs like:
     883
     884                  if initial is None (or in none_equivalents):
     885                          real_value = (template_instead % instead) or instead
     886                  else:
     887                          real_value = (template_initial % initial) or initial
     888                  print real_value
     889
     890          @param initial: the value to be tested for <None>
     891          @type initial: any Python type, must have a __str__ method if template_initial is not None
     892          @param instead: the value to be returned if <initial> is None
     893          @type instead: any Python type, must have a __str__ method if template_instead is not None
     894          @param template_initial: if <initial> is returned replace the value into this template, must contain one <%s>
     895          @type template_initial: string or None
     896          @param template_instead: if <instead> is returned replace the value into this template, must contain one <%s>
     897          @type template_instead: string or None
     898
     899          example:
     900                  function_initial = ('strftime', '%Y-%m-%d')
     901
     902          Ideas:
     903                  - list of insteads: initial, [instead, template], [instead, template], [instead, template], template_initial, ...
     904          """
     905          if none_equivalents is None:
     906                  none_equivalents = [None]
     907
     908          if initial in none_equivalents:
     909
     910                  if template_instead is None:
     911                          return instead
     912
     913                  return template_instead % instead
     914
     915          if function_initial is not None:
     916                  funcname, args = function_initial
     917                  func = getattr(initial, funcname)
     918                  initial = func(args)
     919
     920          if template_initial is None:
     921                  return initial
     922
     923          try:
     924                  return template_initial % initial
     925          except TypeError:
     926                  return template_initial
     927
     928  #---------------------------------------------------------------------------
     929 -def __cap_name(match_obj=None):
     930          val = match_obj.group(0).lower()
     931          if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']:                        # FIXME: this needs to expand, configurable ?
     932                  return val
     933          buf = list(val)
     934          buf[0] = buf[0].upper()
     935          for part in ['mac', 'mc', 'de', 'la']:
     936                  if len(val) > len(part) and val[:len(part)] == part:
     937                          buf[len(part)] = buf[len(part)].upper()
     938          return ''.join(buf)
     939
     940  #---------------------------------------------------------------------------
     941 -def capitalize(text=None, mode=CAPS_NAMES):
     942          """Capitalize the first character but leave the rest alone.
     943
     944          Note that we must be careful about the locale, this may
     945          have issues ! However, for UTF strings it should just work.
     946          """
     947          if (mode is None) or (mode == CAPS_NONE):
     948                  return text
     949
     950          if len(text) == 0:
     951                  return text
     952
     953          if mode == CAPS_FIRST:
     954                  if len(text) == 1:
     955                          return text[0].upper()
     956                  return text[0].upper() + text[1:]
     957
     958          if mode == CAPS_ALLCAPS:
     959                  return text.upper()
     960
     961          if mode == CAPS_FIRST_ONLY:
     962  #               if len(text) == 1:
     963  #                       return text[0].upper()
     964                  return text[0].upper() + text[1:].lower()
     965
     966          if mode == CAPS_WORDS:
     967                  #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
     968                  return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text)
     969
     970          if mode == CAPS_NAMES:
     971                  #return regex.sub(r'\w+', __cap_name, text)
     972                  return capitalize(text=text, mode=CAPS_FIRST)           # until fixed
     973
     974          print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode)
     975          return text
     976
     977  #---------------------------------------------------------------------------
     978 -def input2decimal(initial=None):
     979
     980          if isinstance(initial, decimal.Decimal):
     981                  return True, initial
     982
     983          val = initial
     984
     985          # float ? -> to string first
     986          if type(val) == type(float(1.4)):
     987                  val = str(val)
     988
     989          # string ? -> "," to "."
     990          if isinstance(val, str):
     991                  val = val.replace(',', '.', 1)
     992                  val = val.strip()
     993
     994          try:
     995                  d = decimal.Decimal(val)
     996                  return True, d
     997          except (TypeError, decimal.InvalidOperation):
     998                  return False, val
     999
    1000  #---------------------------------------------------------------------------
    1001 -def input2int(initial=None, minval=None, maxval=None):
    1002
    1003          val = initial
    1004
    1005          # string ? -> "," to "."
    1006          if isinstance(val, str):
    1007                  val = val.replace(',', '.', 1)
    1008                  val = val.strip()
    1009
    1010          try:
    1011                  int_val = int(val)
    1012          except (TypeError, ValueError):
    1013                  _log.exception('int(%s) failed', val)
    1014                  return False, initial
    1015
    1016          if minval is not None:
    1017                  if int_val < minval:
    1018                          _log.debug('%s < min (%s)', val, minval)
    1019                          return False, initial
    1020          if maxval is not None:
    1021                  if int_val > maxval:
    1022                          _log.debug('%s > max (%s)', val, maxval)
    1023                          return False, initial
    1024
    1025          return True, int_val
    1026
    1027  #---------------------------------------------------------------------------
    1028 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
    1029          if remove_repeats:
    1030                  if remove_whitespace:
    1031                          while text.lstrip().startswith(prefix):
    1032                                  text = text.lstrip().replace(prefix, '', 1).lstrip()
    1033                          return text
    1034                  while text.startswith(prefix):
    1035                          text = text.replace(prefix, '', 1)
    1036                  return text
    1037          if remove_whitespace:
    1038                  return text.lstrip().replace(prefix, '', 1).lstrip()
    1039          return text.replace(prefix, '', 1)
    1040
    1041  #---------------------------------------------------------------------------
    1042 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
    1043          suffix_len = len(suffix)
    1044          if remove_repeats:
    1045                  if remove_whitespace:
    1046                          while text.rstrip().endswith(suffix):
    1047                                  text = text.rstrip()[:-suffix_len].rstrip()
    1048                          return text
    1049                  while text.endswith(suffix):
    1050                          text = text[:-suffix_len]
    1051                  return text
    1052          if remove_whitespace:
    1053                  return text.rstrip()[:-suffix_len].rstrip()
    1054          return text[:-suffix_len]
    1055
    1056  #---------------------------------------------------------------------------
    1057 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1058          if lines is None:
    1059                  lines = text.split(eol)
    1060
    1061          while True:
    1062                  if lines[0].strip(eol).strip() != '':
    1063                          break
    1064                  lines = lines[1:]
    1065
    1066          if return_list:
    1067                  return lines
    1068
    1069          return eol.join(lines)
    1070
    1071  #---------------------------------------------------------------------------
    1072 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1073          if lines is None:
    1074                  lines = text.split(eol)
    1075
    1076          while True:
    1077                  if lines[-1].strip(eol).strip() != '':
    1078                          break
    1079                  lines = lines[:-1]
    1080
    1081          if return_list:
    1082                  return lines
    1083
    1084          return eol.join(lines)
    1085
    1086  #---------------------------------------------------------------------------
    1087 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
    1088          return strip_trailing_empty_lines (
    1089                  lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True),
    1090                  text = None,
    1091                  eol = eol,
    1092                  return_list = return_list
    1093          )
    1094
    1095  #---------------------------------------------------------------------------
    1096 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True):
    1097
    1098          if len(lines) == 0:
    1099                  return ''
    1100
    1101          if strip_leading_empty_lines:
    1102                  lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True)
    1103
    1104          if strip_trailing_empty_lines:
    1105                  lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True)
    1106
    1107          if strip_trailing_whitespace:
    1108                  lines = [ l.rstrip() for l in lines ]
    1109
    1110          indented_lines = [initial_indent + lines[0]]
    1111          indented_lines.extend([ subsequent_indent + l for l in lines[1:] ])
    1112
    1113          return eol.join(indented_lines)
    1114
    1115  #---------------------------------------------------------------------------
    1116 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
    1117          """A word-wrap function that preserves existing line breaks
    1118          and most spaces in the text. Expects that existing line
    1119          breaks are posix newlines (\n).
    1120          """
    1121          if width is None:
    1122                  return text
    1123          wrapped = initial_indent + functools.reduce (
    1124                  lambda line, word, width=width: '%s%s%s' % (
    1125                          line,
    1126                          ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)],
    1127                          word
    1128                  ),
    1129                  text.split(' ')
    1130          )
    1131
    1132          if subsequent_indent != '':
    1133                  wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n'))
    1134
    1135          if eol != '\n':
    1136                  wrapped = wrapped.replace('\n', eol)
    1137
    1138          return wrapped
    1139
    1140  #---------------------------------------------------------------------------
    1141 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
    1142
    1143          text = text.replace('\r', '')
    1144          lines = text.split('\n')
    1145          text = ''
    1146          for line in lines:
    1147
    1148                  if strip_whitespace:
    1149                          line = line.strip().strip('\t').strip()
    1150
    1151                  if remove_empty_lines:
    1152                          if line == '':
    1153                                  continue
    1154
    1155                  text += ('%s%s' % (line, line_separator))
    1156
    1157          text = text.rstrip(line_separator)
    1158
    1159          if max_length is not None:
    1160                  text = text[:max_length]
    1161
    1162          text = text.rstrip(line_separator)
    1163
    1164          return text
    1165
    1166  #---------------------------------------------------------------------------
    1167 -def shorten_text(text=None, max_length=None):
    1168
    1169          if len(text) <= max_length:
    1170                  return text
    1171
    1172          return text[:max_length-1] + u_ellipsis
    1173
    1174  #---------------------------------------------------------------------------
    1175 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
    1176          if text is None:
    1177                  return None
    1178          if max_length is None:
    1179                  max_length = len(text)
    1180          else:
    1181                  if len(text) <= max_length:
    1182                          return text
    1183          old_words = regex.split('\s+', text, flags = regex.UNICODE)
    1184          no_old_words = len(old_words)
    1185          max_word_length = max(min_word_length, (max_length // no_old_words))
    1186          words = []
    1187          for word in old_words:
    1188                  if len(word) <= max_word_length:
    1189                          words.append(word)
    1190                          continue
    1191                  if ignore_numbers:
    1192                          tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '')
    1193                          if tmp.isdigit():
    1194                                  words.append(word)
    1195                                  continue
    1196                  words.append(word[:max_word_length] + ellipsis)
    1197          return ' '.join(words)
    1198
    1199  #---------------------------------------------------------------------------
    1200 -def xml_escape_string(text=None):
    1201          """check for special XML characters and transform them"""
    1202          return xml_tools.escape(text)
    1203
    1204  #---------------------------------------------------------------------------
    1205 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
    1206          """Check for special TeX characters and transform them.
    1207
    1208                  replace_eol:
    1209                          replaces "\n" with "\\newline"
    1210                  keep_visual_eol:
    1211                          replaces "\n" with "\\newline \n" such that
    1212                          both LaTeX will know to place a line break
    1213                          at this point as well as the visual formatting
    1214                          is preserved in the LaTeX source (think multi-
    1215                          row table cells)
    1216          """
    1217          text = text.replace('\\', '\\textbackslash')                    # requires \usepackage{textcomp} in LaTeX source
    1218          text = text.replace('^', '\\textasciicircum')
    1219          text = text.replace('~', '\\textasciitilde')
    1220
    1221          text = text.replace('{', '\\{')
    1222          text = text.replace('}', '\\}')
    1223          text = text.replace('%', '\\%')
    1224          text = text.replace('&', '\\&')
    1225          text = text.replace('#', '\\#')
    1226          text = text.replace('$', '\\$')
    1227          text = text.replace('_', '\\_')
    1228          if replace_eol:
    1229                  if keep_visual_eol:
    1230                          text = text.replace('\n', '\\newline \n')
    1231                  else:
    1232                          text = text.replace('\n', '\\newline ')
    1233
    1234          if replace_known_unicode:
    1235                  # this should NOT be replaced for Xe(La)Tex
    1236                  text = text.replace(u_euro, '\\EUR')            # requires \usepackage{textcomp} in LaTeX source
    1237                  text = text.replace(u_sum, '$\\Sigma$')
    1238
    1239          return text
    1240
    1241  #---------------------------------------------------------------------------
    1242 -def rst2latex_snippet(rst_text):
    1243          global du_core
    1244          if du_core is None:
    1245                  try:
    1246                          from docutils import core as du_core
    1247                  except ImportError:
    1248                          _log.warning('cannot turn ReST into LaTeX: docutils not installed')
    1249                          return tex_escape_string(text = rst_text)
    1250
    1251          parts = du_core.publish_parts (
    1252                  source = rst_text.replace('\\', '\\\\'),
    1253                  source_path = '<internal>',
    1254                  writer_name = 'latex',
    1255                  #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
    1256                  settings_overrides = {
    1257                          'input_encoding': 'unicode'             # un-encoded unicode
    1258                  },
    1259                  enable_exit_status = True                       # how to use ?
    1260          )
    1261          return parts['body']
    1262
    1263  #---------------------------------------------------------------------------
    1264 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
    1265          global du_core
    1266          if du_core is None:
    1267                  try:
    1268                          from docutils import core as du_core
    1269                  except ImportError:
    1270                          _log.warning('cannot turn ReST into HTML: docutils not installed')
    1271                          return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False)
    1272
    1273          parts = du_core.publish_parts (
    1274                  source = rst_text.replace('\\', '\\\\'),
    1275                  source_path = '<internal>',
    1276                  writer_name = 'latex',
    1277                  #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex',
    1278                  settings_overrides = {
    1279                          'input_encoding': 'unicode'             # un-encoded unicode
    1280                  },
    1281                  enable_exit_status = True                       # how to use ?
    1282          )
    1283          return parts['body']
    1284
    1285  #---------------------------------------------------------------------------
    1286 -def xetex_escape_string(text=None):
    1287          # a web search did not reveal anything else for Xe(La)Tex
    1288          # as opposed to LaTeX, except true unicode chars
    1289          return tex_escape_string(text = text, replace_known_unicode = False)
    1290
    1291  #---------------------------------------------------------------------------
    1292  __html_escape_table = {
    1293          "&": "&amp;",
    1294          '"': "&quot;",
    1295          "'": "&apos;",
    1296          ">": "&gt;",
    1297          "<": "&lt;",
    1298  }
    1299
    1300 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
    1301          text = ''.join(__html_escape_table.get(char, char) for char in text)
    1302          if replace_eol:
    1303                  if keep_visual_eol:
    1304                          text = text.replace('\n', '<br>\n')
    1305                  else:
    1306                          text = text.replace('\n', '<br>')
    1307          return text
    1308
    1309  #---------------------------------------------------------------------------
    1310 -def dict2json(obj):
    1311          return json.dumps(obj, default = json_serialize)
    1312
    1313  #---------------------------------------------------------------------------
    1314 -def json_serialize(obj):
    1315          if isinstance(obj, pydt.datetime):
    1316                  return obj.isoformat()
    1317          raise TypeError('cannot json_serialize(%s)' % type(obj))
    1318
    1319  #---------------------------------------------------------------------------
    1320  #---------------------------------------------------------------------------
    1321 -def compare_dict_likes(d1, d2, title1=None, title2=None):
    1322          _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2))
    1323          try:
    1324                  d1 = dict(d1)
    1325          except TypeError:
    1326                  pass
    1327          try:
    1328                  d2 = dict(d2)
    1329          except TypeError:
    1330                  pass
    1331          keys_d1 = frozenset(d1.keys())
    1332          keys_d2 = frozenset(d2.keys())
    1333          different = False
    1334          if len(keys_d1) != len(keys_d2):
    1335                  _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2))
    1336                  different = True
    1337          for key in keys_d1:
    1338                  if key in keys_d2:
    1339                          if type(d1[key]) != type(d2[key]):
    1340                                  _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key]))
    1341                                  _log.info('%25.25s  type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key]))
    1342                                  different = True
    1343                                  continue
    1344                          if d1[key] == d2[key]:
    1345                                  _log.info('%25.25s:  both = >>>%s<<<' % (key, d1[key]))
    1346                          else:
    1347                                  _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key]))
    1348                                  _log.info('%25.25s  dict2 = >>>%s<<<' % ('', d2[key]))
    1349                                  different = True
    1350                  else:
    1351                          _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key]))
    1352                          different = True
    1353          for key in keys_d2:
    1354                  if key in keys_d1:
    1355                          continue
    1356                  _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key]))
    1357                  different = True
    1358          if different:
    1359                  _log.info('dict-likes appear to be different from each other')
    1360                  return False
    1361          _log.info('dict-likes appear equal to each other')
    1362          return True
    1363
    1364  #---------------------------------------------------------------------------
    1365 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
    1366
    1367          _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2))
    1368          append_type = False
    1369          if None not in [title_left, title_right]:
    1370                  append_type = True
    1371                  type_left = type(d1)
    1372                  type_right = type(d2)
    1373          if title_left is None:
    1374                  title_left = '%s' % type_left
    1375          if title_right is None:
    1376                  title_right = '%s' % type_right
    1377
    1378          try: d1 = dict(d1)
    1379          except TypeError: pass
    1380          try: d2 = dict(d2)
    1381          except TypeError: pass
    1382          keys_d1 = d1.keys()
    1383          keys_d2 = d2.keys()
    1384          data = {}
    1385          for key in keys_d1:
    1386                  data[key] = [d1[key], ' ']
    1387                  if key in d2:
    1388                          data[key][1] = d2[key]
    1389          for key in keys_d2:
    1390                  if key in keys_d1:
    1391                          continue
    1392                  data[key] = [' ', d2[key]]
    1393          max1 = max([ len('%s' % k) for k in keys_d1 ])
    1394          max2 = max([ len('%s' % k) for k in keys_d2 ])
    1395          max_len = max(max1, max2, len(_('<type>')))
    1396          max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's'
    1397          max1 = max([ len('%s' % d1[k]) for k in keys_d1 ])
    1398          max2 = max([ len('%s' % d2[k]) for k in keys_d2 ])
    1399          max_data_len = min(max(max1, max2), 100)
    1400          max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's'
    1401          diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's'
    1402          line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s'
    1403
    1404          lines = []
    1405          # debugging:
    1406          #lines.append(u'                                        (40 regular spaces)')
    1407          #lines.append((u' ' * 40) + u"(u' ' * 40)")
    1408          #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')")
    1409          #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')")
    1410          #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')")
    1411          #lines.append(line_template)
    1412          lines.append(line_template % ('', '', title_left, title_right))
    1413          if append_type:
    1414                  lines.append(line_template % ('', _('<type>'), type_left, type_right))
    1415
    1416          if ignore_diff_in_keys is None:
    1417                  ignore_diff_in_keys = []
    1418
    1419          for key in keys_d1:
    1420                  append_type = False
    1421                  txt_left_col = '%s' % d1[key]
    1422                  try:
    1423                          txt_right_col = '%s' % d2[key]
    1424                          if type(d1[key]) != type(d2[key]):
    1425                                  append_type = True
    1426                  except KeyError:
    1427                          txt_right_col = missing_string
    1428                  lines.append(line_template % (
    1429                          bool2subst (
    1430                                  ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)),
    1431                                  '',
    1432                                  difference_indicator
    1433                          ),
    1434                          key,
    1435                          shorten_text(txt_left_col, max_data_len),
    1436                          shorten_text(txt_right_col, max_data_len)
    1437                  ))
    1438                  if append_type:
    1439                          lines.append(line_template % (
    1440                                  '',
    1441                                  _('<type>'),
    1442                                  shorten_text('%s' % type(d1[key]), max_data_len),
    1443                                  shorten_text('%s' % type(d2[key]), max_data_len)
    1444                          ))
    1445
    1446          for key in keys_d2:
    1447                  if key in keys_d1:
    1448                          continue
    1449                  lines.append(line_template % (
    1450                          bool2subst((key in ignore_diff_in_keys), '', difference_indicator),
    1451                          key,
    1452                          shorten_text(missing_string, max_data_len),
    1453                          shorten_text('%s' % d2[key], max_data_len)
    1454                  ))
    1455
    1456          return lines
    1457
    1458  #---------------------------------------------------------------------------
    1459 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
    1460          if values2ignore is None:
    1461                  values2ignore = []
    1462          if template is not None:
    1463                  # all keys in template better exist in d
    1464                  try:
    1465                          return template % d
    1466                  except KeyError:
    1467                          # or else
    1468                          _log.exception('template contains %%()s key(s) which do not exist in data dict')
    1469                  # try to extend dict <d> to contain all required keys,
    1470                  # for that to work <relevant_keys> better list all
    1471                  # keys used in <template>
    1472                  if relevant_keys is not None:
    1473                          for key in relevant_keys:
    1474                                  try:
    1475                                          d[key]
    1476                                  except KeyError:
    1477                                          d[key] = missing_key_template % {'key': key}
    1478                          return template % d
    1479
    1480          if relevant_keys is None:
    1481                  relevant_keys = list(d.keys())
    1482          lines = []
    1483          if value_delimiters is None:
    1484                  delim_left = ''
    1485                  delim_right = ''
    1486          else:
    1487                  delim_left, delim_right = value_delimiters
    1488          if tabular:
    1489                  max_len = max([ len('%s' % k) for k in relevant_keys ])
    1490                  max_len_str = '%s.%s' % (max_len, max_len)
    1491                  line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right))
    1492          else:
    1493                  line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right)
    1494          for key in relevant_keys:
    1495                  try:
    1496                          val = d[key]
    1497                  except KeyError:
    1498                          continue
    1499                  if val not in values2ignore:
    1500                          lines.append(line_template % (key, val))
    1501          if eol is None:
    1502                  return lines
    1503          return eol.join(lines)
    1504
    1505  #---------------------------------------------------------------------------
    1506 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
    1507          for key in required_keys:
    1508                  try:
    1509                          d[key]
    1510                  except KeyError:
    1511                          if missing_key_template is None:
    1512                                  d[key] = None
    1513                          else:
    1514                                  d[key] = missing_key_template % {'key': key}
    1515          return d
    1516
    1517  #---------------------------------------------------------------------------
    1518  #---------------------------------------------------------------------------
    1519 -def prompted_input(prompt=None, default=None):
    1520          """Obtains entry from standard input.
    1521
    1522          prompt: Prompt text to display in standard output
    1523          default: Default value (for user to press enter only)
    1524          CTRL-C: aborts and returns None
    1525          """
    1526          if prompt is None:
    1527                  msg = '(CTRL-C aborts)'
    1528          else:
    1529                  msg = '%s (CTRL-C aborts)' % prompt
    1530
    1531          if default is None:
    1532                  msg = msg + ': '
    1533          else:
    1534                  msg = '%s [%s]: ' % (msg, default)
    1535
    1536          try:
    1537                  usr_input = input(msg)
    1538          except KeyboardInterrupt:
    1539                  return None
    1540
    1541          if usr_input == '':
    1542                  return default
    1543
    1544          return usr_input
    1545
    1546  #===========================================================================
    1547  # image handling tools
    1548  #---------------------------------------------------------------------------
    1549  # builtin (ugly but tried and true) fallback icon
    1550  __icon_serpent = \
    1551  """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\
    1552  \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\
    1553  \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\
    1554  \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\
    1555  \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\
    1556  \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\
    1557  \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec"""
    1558
    1559 -def get_icon(wx=None):
    1560
    1561          paths = gmPaths(app_name = 'gnumed', wx = wx)
    1562
    1563          candidates = [
    1564                  os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
    1565                  os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'),
    1566                  os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'),
    1567                  os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png')
    1568          ]
    1569
    1570          found_as = None
    1571          for candidate in candidates:
    1572                  try:
    1573                          open(candidate, 'r').close()
    1574                          found_as = candidate
    1575                          break
    1576                  except IOError:
    1577                          _log.debug('icon not found in [%s]', candidate)
    1578
    1579          if found_as is None:
    1580                  _log.warning('no icon file found, falling back to builtin (ugly) icon')
    1581                  icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent)))
    1582                  icon.CopyFromBitmap(icon_bmp_data)
    1583          else:
    1584                  _log.debug('icon found in [%s]', found_as)
    1585                  icon = wx.Icon()
    1586                  try:
    1587                          icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY)             #_PNG
    1588                  except AttributeError:
    1589                          _log.exception("this platform doesn't support wx.Icon().LoadFile()")
    1590
    1591          return icon
    1592
    1593  #---------------------------------------------------------------------------
    1594 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
    1595          assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified'
    1596
    1597          try:
    1598                  import pyqrcode
    1599          except ImportError:
    1600                  _log.exception('cannot import <pyqrcode>')
    1601                  return None
    1602          if text is None:
    1603                  with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file:
    1604                          text = input_file.read()
    1605          if qr_filename is None:
    1606                  if filename is None:
    1607                          qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png')
    1608                  else:
    1609                          qr_filename = get_unique_filename (
    1610                                  prefix = fname_stem(filename) + '-',
    1611                                  suffix = fname_extension(filename) + '.png'
    1612                          )
    1613          _log.debug('[%s] -> [%s]', filename, qr_filename)
    1614          qr = pyqrcode.create(text, encoding = 'utf8')
    1615          if verbose:
    1616                  print('input file:', filename)
    1617                  print('output file:', qr_filename)
    1618                  print('text to encode:', text)
    1619                  print(qr.terminal())
    1620          qr.png(qr_filename, quiet_zone = 1)
    1621          return qr_filename
    1622
    1623  #===========================================================================
    1624  # main
    1625  #---------------------------------------------------------------------------
    1626  if __name__ == '__main__':
    1627
    1628          if len(sys.argv) < 2:
    1629                  sys.exit()
    1630
    1631          if sys.argv[1] != 'test':
    1632                  sys.exit()
    1633
    1634          # for testing:
    1635          logging.basicConfig(level = logging.DEBUG)
    1636          from Gnumed.pycommon import gmI18N
    1637          gmI18N.activate_locale()
    1638          gmI18N.install_domain()
    1639
    1640          #-----------------------------------------------------------------------
    1641 -        def test_input2decimal():
    1642
    1643                  tests = [
    1644                          [None, False],
    1645
    1646                          ['', False],
    1647                          [' 0 ', True, 0],
    1648
    1649                          [0, True, 0],
    1650                          [0.0, True, 0],
    1651                          [.0, True, 0],
    1652                          ['0', True, 0],
    1653                          ['0.0', True, 0],
    1654                          ['0,0', True, 0],
    1655                          ['00.0', True, 0],
    1656                          ['.0', True, 0],
    1657                          [',0', True, 0],
    1658
    1659                          [0.1, True, decimal.Decimal('0.1')],
    1660                          [.01, True, decimal.Decimal('0.01')],
    1661                          ['0.1', True, decimal.Decimal('0.1')],
    1662                          ['0,1', True, decimal.Decimal('0.1')],
    1663                          ['00.1', True, decimal.Decimal('0.1')],
    1664                          ['.1', True, decimal.Decimal('0.1')],
    1665                          [',1', True, decimal.Decimal('0.1')],
    1666
    1667                          [1, True, 1],
    1668                          [1.0, True, 1],
    1669                          ['1', True, 1],
    1670                          ['1.', True, 1],
    1671                          ['1,', True, 1],
    1672                          ['1.0', True, 1],
    1673                          ['1,0', True, 1],
    1674                          ['01.0', True, 1],
    1675                          ['01,0', True, 1],
    1676                          [' 01, ', True, 1],
    1677
    1678                          [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')]
    1679                  ]
    1680                  for test in tests:
    1681                          conversion_worked, result = input2decimal(initial = test[0])
    1682
    1683                          expected2work = test[1]
    1684
    1685                          if conversion_worked:
    1686                                  if expected2work:
    1687                                          if result == test[2]:
    1688                                                  continue
    1689                                          else:
    1690                                                  print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result))
    1691                                  else:
    1692                                          print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result))
    1693                          else:
    1694                                  if not expected2work:
    1695                                          continue
    1696                                  else:
    1697                                          print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
    1698          #-----------------------------------------------------------------------
    1699 -        def test_input2int():
    1700                  print(input2int(0))
    1701                  print(input2int('0'))
    1702                  print(input2int('0', 0, 0))
    1703          #-----------------------------------------------------------------------
    1704 -        def test_coalesce():
    1705
    1706                  val = None
    1707                  print(val, coalesce(val, 'is None', 'is not None'))
    1708                  val = 1
    1709                  print(val, coalesce(val, 'is None', 'is not None'))
    1710                  return
    1711
    1712                  import datetime as dt
    1713                  print(coalesce(initial = dt.datetime.now(), template_initial = '-- %s --', function_initial = ('strftime', '%Y-%m-%d')))
    1714
    1715                  print('testing coalesce()')
    1716                  print("------------------")
    1717                  tests = [
    1718                          [None, 'something other than <None>', None, None, 'something other than <None>'],
    1719                          ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'],
    1720                          ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'],
    1721                          ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'],
    1722                          [None, 'initial value was None', 'template_initial: %s', None, 'initial value was None'],
    1723                          [None, 'initial value was None', 'template_initial: %%(abc)s', None, 'initial value was None']
    1724                  ]
    1725                  passed = True
    1726                  for test in tests:
    1727                          result = coalesce (
    1728                                  initial = test[0],
    1729                                  instead = test[1],
    1730                                  template_initial = test[2],
    1731                                  template_instead = test[3]
    1732                          )
    1733                          if result != test[4]:
    1734                                  print("ERROR")
    1735                                  print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3]))
    1736                                  print("expected:", test[4])
    1737                                  print("received:", result)
    1738                                  passed = False
    1739
    1740                  if passed:
    1741                          print("passed")
    1742                  else:
    1743                          print("failed")
    1744                  return passed
    1745          #-----------------------------------------------------------------------
    1746 -        def test_capitalize():
    1747                  print('testing capitalize() ...')
    1748                  success = True
    1749                  pairs = [
    1750                          # [original, expected result, CAPS mode]
    1751                          ['Boot', 'Boot', CAPS_FIRST_ONLY],
    1752                          ['boot', 'Boot', CAPS_FIRST_ONLY],
    1753                          ['booT', 'Boot', CAPS_FIRST_ONLY],
    1754                          ['BoOt', 'Boot', CAPS_FIRST_ONLY],
    1755                          ['boots-Schau', 'Boots-Schau', CAPS_WORDS],
    1756                          ['boots-sChau', 'Boots-Schau', CAPS_WORDS],
    1757                          ['boot camp', 'Boot Camp', CAPS_WORDS],
    1758                          ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES],
    1759                          ['häkkönen', 'Häkkönen', CAPS_NAMES],
    1760                          ['McBurney', 'McBurney', CAPS_NAMES],
    1761                          ['mcBurney', 'McBurney', CAPS_NAMES],
    1762                          ['blumberg', 'Blumberg', CAPS_NAMES],
    1763                          ['roVsing', 'RoVsing', CAPS_NAMES],
    1764                          ['Özdemir', 'Özdemir', CAPS_NAMES],
    1765                          ['özdemir', 'Özdemir', CAPS_NAMES],
    1766                  ]
    1767                  for pair in pairs:
    1768                          result = capitalize(pair[0], pair[2])
    1769                          if result != pair[1]:
    1770                                  success = False
    1771                                  print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1]))
    1772
    1773                  if success:
    1774                          print("... SUCCESS")
    1775
    1776                  return success
    1777          #-----------------------------------------------------------------------
    1778 -        def test_import_module():
    1779                  print("testing import_module_from_directory()")
    1780                  path = sys.argv[1]
    1781                  name = sys.argv[2]
    1782                  try:
    1783                          mod = import_module_from_directory(module_path = path, module_name = name)
    1784                  except:
    1785                          print("module import failed, see log")
    1786                          return False
    1787
    1788                  print("module import succeeded", mod)
    1789                  print(dir(mod))
    1790                  return True
    1791          #-----------------------------------------------------------------------
    1792 -        def test_mkdir():
    1793                  print("testing mkdir(%s)" % sys.argv[2])
    1794                  mkdir(sys.argv[2])
    1795          #-----------------------------------------------------------------------
    1796 -        def test_gmPaths():
    1797                  print("testing gmPaths()")
    1798                  print("-----------------")
    1799                  paths = gmPaths(wx=None, app_name='gnumed')
    1800                  print("user     config dir:", paths.user_config_dir)
    1801                  print("system   config dir:", paths.system_config_dir)
    1802                  print("local      base dir:", paths.local_base_dir)
    1803                  print("system app data dir:", paths.system_app_data_dir)
    1804                  print("working directory  :", paths.working_dir)
    1805                  print("temp directory     :", paths.tmp_dir)
    1806          #-----------------------------------------------------------------------
    1807 -        def test_none_if():
    1808                  print("testing none_if()")
    1809                  print("-----------------")
    1810                  tests = [
    1811                          [None, None, None],
    1812                          ['a', 'a', None],
    1813                          ['a', 'b', 'a'],
    1814                          ['a', None, 'a'],
    1815                          [None, 'a', None],
    1816                          [1, 1, None],
    1817                          [1, 2, 1],
    1818                          [1, None, 1],
    1819                          [None, 1, None]
    1820                  ]
    1821
    1822                  for test in tests:
    1823                          if none_if(value = test[0], none_equivalent = test[1]) != test[2]:
    1824                                  print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2]))
    1825
    1826                  return True
    1827          #-----------------------------------------------------------------------
    1828 -        def test_bool2str():
    1829                  tests = [
    1830                          [True, 'Yes', 'Yes', 'Yes'],
    1831                          [False, 'OK', 'not OK', 'not OK']
    1832                  ]
    1833                  for test in tests:
    1834                          if bool2str(test[0], test[1], test[2]) != test[3]:
    1835                                  print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3]))
    1836
    1837                  return True
    1838          #-----------------------------------------------------------------------
    1839 -        def test_bool2subst():
    1840
    1841                  print(bool2subst(True, 'True', 'False', 'is None'))
    1842                  print(bool2subst(False, 'True', 'False', 'is None'))
    1843                  print(bool2subst(None, 'True', 'False', 'is None'))
    1844          #-----------------------------------------------------------------------
    1845 -        def test_get_unique_filename():
    1846                  print(get_unique_filename())
    1847                  print(get_unique_filename(prefix='test-'))
    1848                  print(get_unique_filename(suffix='tst'))
    1849                  print(get_unique_filename(prefix='test-', suffix='tst'))
    1850                  print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
    1851          #-----------------------------------------------------------------------
    1852 -        def test_size2str():
    1853                  print("testing size2str()")
    1854                  print("------------------")
    1855                  tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000]
    1856                  for test in tests:
    1857                          print(size2str(test))
    1858          #-----------------------------------------------------------------------
    1859 -        def test_unwrap():
    1860
    1861                  test = """
    1862  second line\n
    1863          3rd starts with tab  \n
    1864   4th with a space       \n
    1865
    1866  6th
    1867
    1868  """
    1869                  print(unwrap(text = test, max_length = 25))
    1870          #-----------------------------------------------------------------------
    1871 -        def test_wrap():
    1872                  test = 'line 1\nline 2\nline 3'
    1873
    1874                  print("wrap 5-6-7 initial 0, subsequent 0")
    1875                  print(wrap(test, 5))
    1876                  print()
    1877                  print(wrap(test, 6))
    1878                  print()
    1879                  print(wrap(test, 7))
    1880                  print("-------")
    1881                  input()
    1882                  print("wrap 5 initial 1-1-3, subsequent 1-3-1")
    1883                  print(wrap(test, 5, ' ', ' '))
    1884                  print()
    1885                  print(wrap(test, 5, ' ', '   '))
    1886                  print()
    1887                  print(wrap(test, 5, '   ', ' '))
    1888                  print("-------")
    1889                  input()
    1890                  print("wrap 6 initial 1-1-3, subsequent 1-3-1")
    1891                  print(wrap(test, 6, ' ', ' '))
    1892                  print()
    1893                  print(wrap(test, 6, ' ', '   '))
    1894                  print()
    1895                  print(wrap(test, 6, '   ', ' '))
    1896                  print("-------")
    1897                  input()
    1898                  print("wrap 7 initial 1-1-3, subsequent 1-3-1")
    1899                  print(wrap(test, 7, ' ', ' '))
    1900                  print()
    1901                  print(wrap(test, 7, ' ', '   '))
    1902                  print()
    1903                  print(wrap(test, 7, '   ', ' '))
    1904          #-----------------------------------------------------------------------
    1905 -        def test_md5():
    1906                  print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2])))
    1907                  print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
    1908          #-----------------------------------------------------------------------
    1909 -        def test_unicode():
    1910                  print(u_link_symbol * 10)
    1911          #-----------------------------------------------------------------------
    1912 -        def test_xml_escape():
    1913                  print(xml_escape_string('<'))
    1914                  print(xml_escape_string('>'))
    1915                  print(xml_escape_string('&'))
    1916          #-----------------------------------------------------------------------
    1917 -        def test_tex_escape():
    1918                  tests = ['\\', '^', '~', '{', '}', '%',  '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
    1919                  tests.append('  '.join(tests))
    1920                  for test in tests:
    1921                          print('%s:' % test, tex_escape_string(test))
    1922
    1923          #-----------------------------------------------------------------------
    1924 -        def test_rst2latex_snippet():
    1925                  tests = ['\\', '^', '~', '{', '}', '%',  '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234']
    1926                  tests.append('  '.join(tests))
    1927                  tests.append('C:\Windows\Programme\System 32\lala.txt')
    1928                  tests.extend([
    1929                          'should be identical',
    1930                          'text *some text* text',
    1931                          """A List
    1932  ======
    1933
    1934  1. 1
    1935  2. 2
    1936
    1937  3. ist-list
    1938  1. more
    1939  2. noch was ü
    1940  #. nummer x"""
    1941                  ])
    1942                  for test in tests:
    1943                          print('==================================================')
    1944                          print('raw:')
    1945                          print(test)
    1946                          print('---------')
    1947                          print('ReST 2 LaTeX:')
    1948                          latex = rst2latex_snippet(test)
    1949                          print(latex)
    1950                          if latex.strip() == test.strip():
    1951                                  print('=> identical')
    1952                          print('---------')
    1953                          print('tex_escape_string:')
    1954                          print(tex_escape_string(test))
    1955                          input()
    1956
    1957          #-----------------------------------------------------------------------
    1958 -        def test_strip_trailing_empty_lines():
    1959                  tests = [
    1960                          'one line, no embedded line breaks  ',
    1961                          'one line\nwith embedded\nline\nbreaks\n   '
    1962                  ]
    1963                  for test in tests:
    1964                          print('as list:')
    1965                          print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True))
    1966                          print('as string:')
    1967                          print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False))
    1968                  tests = [
    1969                          ['list', 'without', 'empty', 'trailing', 'lines'],
    1970                          ['list', 'with', 'empty', 'trailing', 'lines', '', '  ', '']
    1971                  ]
    1972                  for test in tests:
    1973                          print('as list:')
    1974                          print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True))
    1975                          print('as string:')
    1976                          print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
    1977          #-----------------------------------------------------------------------
    1978 -        def test_fname_stem():
    1979                  tests = [
    1980                          r'abc.exe',
    1981                          r'\abc.exe',
    1982                          r'c:\abc.exe',
    1983                          r'c:\d\abc.exe',
    1984                          r'/home/ncq/tmp.txt',
    1985                          r'~/tmp.txt',
    1986                          r'./tmp.txt',
    1987                          r'./.././tmp.txt',
    1988                          r'tmp.txt'
    1989                  ]
    1990                  for t in tests:
    1991                          print("[%s] -> [%s]" % (t, fname_stem(t)))
    1992          #-----------------------------------------------------------------------
    1993 -        def test_dir_is_empty():
    1994                  print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
    1995
    1996          #-----------------------------------------------------------------------
    1997 -        def test_compare_dicts():
    1998                  d1 = {}
    1999                  d2 = {}
    2000                  d1[1] = 1
    2001                  d1[2] = 2
    2002                  d1[3] = 3
    2003                  # 4
    2004                  d1[5] = 5
    2005
    2006                  d2[1] = 1
    2007                  d2[2] = None
    2008                  # 3
    2009                  d2[4] = 4
    2010
    2011                  #compare_dict_likes(d1, d2)
    2012
    2013                  d1 = {1: 1, 2: 2}
    2014                  d2 = {1: 1, 2: 2}
    2015
    2016                  #compare_dict_likes(d1, d2, 'same1', 'same2')
    2017                  print(format_dict_like(d1, tabular = False))
    2018                  print(format_dict_like(d1, tabular = True))
    2019                  #print(format_dict_like(d2))
    2020
    2021          #-----------------------------------------------------------------------
    2022 -        def test_format_compare_dicts():
    2023                  d1 = {}
    2024                  d2 = {}
    2025                  d1[1] = 1
    2026                  d1[2] = 2
    2027                  d1[3] = 3
    2028                  # 4
    2029                  d1[5] = 5
    2030
    2031                  d2[1] = 1
    2032                  d2[2] = None
    2033                  # 3
    2034                  d2[4] = 4
    2035
    2036                  print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
    2037
    2038                  d1 = {1: 1, 2: 2}
    2039                  d2 = {1: 1, 2: 2}
    2040
    2041                  print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
    2042
    2043          #-----------------------------------------------------------------------
    2044 -        def test_rm_dir():
    2045                  rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
    2046
    2047          #-----------------------------------------------------------------------
    2048 -        def test_rm_dir_content():
    2049                  #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx'))
    2050                  print(rm_dir_content('/tmp/user/1000/tmp'))
    2051
    2052          #-----------------------------------------------------------------------
    2053 -        def test_strip_prefix():
    2054                  tests = [
    2055                          ('', '', ''),
    2056                          ('a', 'a', ''),
    2057                          ('\.br\MICROCYTES+1\.br\SPHEROCYTES       present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES       present\.br\POLYCHROMASIAmoderate\.br\\')
    2058                  ]
    2059                  for test in tests:
    2060                          text, prefix, expect = test
    2061                          result = strip_prefix(text, prefix)
    2062                          if result == expect:
    2063                                  continue
    2064                          print('test failed:', test)
    2065                          print('result:', result)
    2066          #-----------------------------------------------------------------------
    2067 -        def test_shorten_text():
    2068                  tst = [
    2069                          ('123', 1),
    2070                          ('123', 2),
    2071                          ('123', 3),
    2072                          ('123', 4),
    2073                          ('', 1),
    2074                          ('1', 1),
    2075                          ('12', 1),
    2076                          ('', 2),
    2077                          ('1', 2),
    2078                          ('12', 2),
    2079                          ('123', 2)
    2080                  ]
    2081                  for txt, lng in tst:
    2082                          print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
    2083          #-----------------------------------------------------------------------
    2084 -        def test_fname_sanitize():
    2085                  tests = [
    2086                          '/tmp/test.txt',
    2087                          '/tmp/ test.txt',
    2088                          '/tmp/ tes\\t.txt',
    2089                          'test'
    2090                  ]
    2091                  for test in tests:
    2092                          print (test, fname_sanitize(test))
    2093
    2094          #-----------------------------------------------------------------------
    2095 -        def test_create_qrcode():
    2096                  print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
    2097
    2098          #-----------------------------------------------------------------------
    2099          #test_coalesce()
    2100          #test_capitalize()
    2101          #test_import_module()
    2102          #test_mkdir()
    2103          #test_gmPaths()
    2104          #test_none_if()
    2105          #test_bool2str()
    2106          #test_bool2subst()
    2107          #test_get_unique_filename()
    2108          #test_size2str()
    2109          #test_wrap()
    2110          #test_input2decimal()
    2111          #test_input2int()
    2112          #test_unwrap()
    2113          #test_md5()
    2114          #test_unicode()
    2115          #test_xml_escape()
    2116          #test_strip_trailing_empty_lines()
    2117          #test_fname_stem()
    2118          #test_tex_escape()
    2119          #test_rst2latex_snippet()
    2120          #test_dir_is_empty()
    2121          #test_compare_dicts()
    2122          #test_rm_dir()
    2123          #test_rm_dir_content()
    2124          #test_strip_prefix()
    2125          #test_shorten_text()
    2126          #test_format_compare_dicts()
    2127          #test_fname_sanitize()
    2128          #test_create_qrcode()
    2129
    2130  #===========================================================================
    2131