Gnumed.business.gmHL7

Package Gnumed :: Package business :: Module gmHL7 |
[frames] | no frames]

# Source Code for Module Gnumed.business.gmHL7

       1  # -*- coding: utf-8 -*-
       2  """Some HL7 handling."""
       3  #============================================================
       4  __author__ = "K.Hilbert <Karsten.Hilbert@gmx.net>"
       5  __license__ = "GPL v2 or later"
       6
       7
       8  import sys
       9  import os
      10  import io
      11  import logging
      12  import time
      13  import shutil
      14  import datetime as pyDT
      15  import hl7 as pyhl7
      16  from xml.etree import ElementTree as pyxml
      17
      18
      19  if __name__ == '__main__':
      20          sys.path.insert(0, '../../')
      21
      22  from Gnumed.pycommon import gmI18N
      23  if __name__ == '__main__':
      24          gmI18N.activate_locale()
      25          gmI18N.install_domain()
      26  from Gnumed.pycommon import gmTools
      27  from Gnumed.pycommon import gmBusinessDBObject
      28  from Gnumed.pycommon import gmPG2
      29  from Gnumed.pycommon import gmDateTime
      30
      31  from Gnumed.business import gmIncomingData
      32  from Gnumed.business import gmPathLab
      33  from Gnumed.business import gmPerson
      34  from Gnumed.business import gmPraxis
      35  from Gnumed.business import gmStaff
      36
      37
      38  _log = logging.getLogger('gm.hl7')
      39
      40  # constants
      41  HL7_EOL = '\r'
      42  HL7_BRK = '\.br\\'
      43
      44  HL7_SEGMENTS = 'FHS BHS MSH PID PV1 OBX NTE ORC OBR'.split()
      45
      46  HL7_segment2field_count = {
      47          'FHS': 12,
      48          'BHS': 12,
      49          'MSH': 19,
      50          'PID': 30,
      51          'PV1': 52,
      52          'OBR': 43,
      53          'OBX': 17,
      54          'NTE': 3,
      55          'ORC': 19
      56  }
      57
      58  MSH_field__sending_lab = 3
      59
      60  PID_field__name = 5
      61  PID_field__dob = 7
      62  PID_field__gender = 8
      63  PID_component__lastname = 1
      64  PID_component__firstname = 2
      65  PID_component__middlename = 3
      66
      67  OBR_field__service_name = 4
      68  OBR_field__ts_requested = 6
      69  OBR_field__ts_started = 7
      70  OBR_field__ts_ended = 8
      71  OBR_field__ts_specimen_received = 14
      72
      73  OBX_field__set_id = 1
      74  OBX_field__datatype = 2
      75  OBX_field__type = 3
      76  # components of 3rd field:
      77  OBX_component__loinc = 1
      78  OBX_component__name = 2
      79  OBX_field__subid = 4
      80  OBX_field__value = 5
      81  OBX_field__unit = 6
      82  OBX_field__range = 7
      83  OBX_field__abnormal_flag = 8
      84  OBX_field__status = 11
      85  OBX_field__timestamp = 14
      86
      87  NET_field__set_id = 1
      88  NET_field__src = 2
      89  NET_field__note = 3
      90
      91  HL7_field_labels = {
      92          'MSH': {
      93                  0: 'Segment Type',
      94                  1: 'Field Separator',
      95                  2: 'Encoding Characters',
      96                  3: 'Sending Application',
      97                  4: 'Sending Facility',
      98                  5: 'Receiving Application',
      99                  6: 'Receiving Facility',
     100                  7: 'Date/Time of Message',
     101                  8: 'Security',
     102                  9: 'Message Type',
     103                  10: 'ID: Message Control',
     104                  11: 'ID: Processing',
     105                  12: 'ID: Version',
     106                  14: 'Continuation Pointer',
     107                  15: 'Accept Acknowledgement Type',
     108                  16: 'Application Acknowledgement Type'
     109          },
     110          'PID': {
     111                  0: 'Segment Type',
     112                  1: '<PID> Set ID',
     113                  2: 'Patient ID (external)',
     114                  3: 'Patient ID (internal)',
     115                  4: 'Patient ID (alternate)',
     116                  5: 'Patient Name',
     117                  7: 'Date/Time of birth',
     118                  8: 'Administrative Gender',
     119                  11: 'Patient Address',
     120                  13: 'Patient Phone Number - Home'
     121          },
     122          'OBR': {
     123                  0: 'Segment Type',
     124                  1: 'ID: Set',
     125                  3: 'Filler Order Number (= ORC-3)',
     126                  4: 'ID: Universal Service',
     127                  5: 'Priority',
     128                  6: 'Date/Time requested',
     129                  7: 'Date/Time Observation started',
     130                  14: 'Date/Time Specimen received',
     131                  16: 'Ordering Provider',
     132                  18: 'Placer Field 1',
     133                  20: 'Filler Field 1',
     134                  21: 'Filler Field 2',
     135                  22: 'Date/Time Results reported/Status changed',
     136                  24: 'ID: Diagnostic Service Section',
     137                  25: 'Result Status',
     138                  27: 'Quantity/Timing',
     139                  28: 'Result Copies To'
     140          },
     141          'ORC': {
     142                  0: 'Segment Type',
     143                  1: 'Order Control',
     144                  3: 'Filler Order Number',
     145                  12: 'Ordering Provider'
     146          },
     147          'OBX': {
     148                  0: 'Segment Type',
     149                  1: 'Set ID',
     150                  2: 'Value Type',
     151                  3: 'Identifier (LOINC)',
     152                  4: 'Observation Sub-ID',
     153                  5: 'Value',
     154                  6: 'Units',
     155                  7: 'References Range (Low - High)',
     156                  8: 'Abnormal Flags',
     157                  11: 'Result Status',
     158                  14: 'Date/Time of Observation'
     159          },
     160          'NTE': {
     161                  0: 'Segment Type',
     162                  3: 'Comment'
     163          }
     164  }
     165
     166  HL7_GENDERS = {
     167          'F': 'f',
     168          'M': 'm',
     169          'O': None,
     170          'U': None,
     171          None: None
     172  }
     173
     174  #============================================================
     175  # public API
     176  #============================================================
     177 -def extract_HL7_from_XML_CDATA(filename, xml_path, target_dir=None):
     178
     179          _log.debug('extracting HL7 from CDATA of <%s> nodes in XML file [%s]', xml_path, filename)
     180
     181          # sanity checks/setup
     182          try:
     183                  open(filename).close()
     184                  orig_dir = os.path.split(filename)[0]
     185                  work_filename = gmTools.get_unique_filename(prefix = 'gm-x2h-%s-' % gmTools.fname_stem(filename), suffix = '.hl7')
     186                  if target_dir is None:
     187                          target_dir = os.path.join(orig_dir, 'HL7')
     188                          done_dir = os.path.join(orig_dir, 'done')
     189                  else:
     190                          done_dir = os.path.join(target_dir, 'done')
     191                  _log.debug('target dir: %s', target_dir)
     192                  gmTools.mkdir(target_dir)
     193                  gmTools.mkdir(done_dir)
     194          except Exception:
     195                  _log.exception('cannot setup unwrapping environment')
     196                  return None
     197
     198          hl7_xml = pyxml.ElementTree()
     199          try:
     200                  hl7_xml.parse(filename)
     201          except pyxml.ParseError:
     202                  _log.exception('cannot parse [%s]' % filename)
     203                  return None
     204          nodes = hl7_xml.findall(xml_path)
     205          if len(nodes) == 0:
     206                  _log.debug('no nodes found for data extraction')
     207                  return None
     208
     209          _log.debug('unwrapping HL7 from XML into [%s]', work_filename)
     210          hl7_file = io.open(work_filename, mode = 'wt', encoding = 'utf8', newline = '')         # universal newlines acceptance but no translation on output
     211          for node in nodes:
     212  #               hl7_file.write(node.text.rstrip() + HL7_EOL)
     213                  hl7_file.write(node.text + '')          # trick to make node.text unicode
     214          hl7_file.close()
     215
     216          target_fname = os.path.join(target_dir, os.path.split(work_filename)[1])
     217          shutil.copy(work_filename, target_dir)
     218          shutil.move(filename, done_dir)
     219
     220          return target_fname
     221
     222  #------------------------------------------------------------
     223 -def split_hl7_file(filename, target_dir=None, encoding='utf8'):
     224          """Multi-step processing of HL7 files.
     225
     226          - input can be multi-MSH / multi-PID / partially malformed HL7
     227          - tries to fix oddities
     228          - splits by MSH
     229          - splits by PID into <target_dir>
     230
     231          - needs write permissions in dir_of(filename)
     232          - moves HL7 files which were successfully split up into dir_of(filename)/done/
     233
     234          - returns (True|False, list_of_PID_files)
     235          """
     236          local_log_name = gmTools.get_unique_filename (
     237                  prefix = gmTools.fname_stem(filename) + '-',
     238                  suffix = '.split.log'
     239          )
     240          local_logger = logging.FileHandler(local_log_name)
     241          local_logger.setLevel(logging.DEBUG)
     242          root_logger = logging.getLogger('')
     243          root_logger.addHandler(local_logger)
     244          _log.info('splitting HL7 file: %s', filename)
     245          _log.debug('log file: %s', local_log_name)
     246
     247          # sanity checks/setup
     248          try:
     249                  open(filename).close()
     250                  orig_dir = os.path.split(filename)[0]
     251                  done_dir = os.path.join(orig_dir, 'done')
     252                  gmTools.mkdir(done_dir)
     253                  error_dir = os.path.join(orig_dir, 'failed')
     254                  gmTools.mkdir(error_dir)
     255                  work_filename = gmTools.get_unique_filename(prefix = gmTools.fname_stem(filename) + '-', suffix = '.hl7')
     256                  if target_dir is None:
     257                          target_dir = os.path.join(orig_dir, 'PID')
     258                  _log.debug('target dir: %s', target_dir)
     259                  gmTools.mkdir(target_dir)
     260          except Exception:
     261                  _log.exception('cannot setup splitting environment')
     262                  root_logger.removeHandler(local_logger)
     263                  return False, None
     264
     265          # split
     266          target_names = []
     267          try:
     268                  shutil.copy(filename, work_filename)
     269                  fixed_filename = __fix_malformed_hl7_file(work_filename, encoding = encoding)
     270                  MSH_fnames = __split_hl7_file_by_MSH(fixed_filename, encoding)
     271                  PID_fnames = []
     272                  for MSH_fname in MSH_fnames:
     273                          PID_fnames.extend(__split_MSH_by_PID(MSH_fname))
     274                  for PID_fname in PID_fnames:
     275                          shutil.move(PID_fname, target_dir)
     276                          target_names.append(os.path.join(target_dir, os.path.split(PID_fname)[1]))
     277          except Exception:
     278                  _log.exception('cannot split HL7 file')
     279                  for target_name in target_names:
     280                          try: os.remove(target_name)
     281                          except: pass
     282                  root_logger.removeHandler(local_logger)
     283                  shutil.move(local_log_name, error_dir)
     284                  return False, None
     285
     286          _log.info('successfully split')
     287          root_logger.removeHandler(local_logger)
     288          try:
     289                  shutil.move(filename, done_dir)
     290                  shutil.move(local_log_name, done_dir)
     291          except shutil.Error:
     292                  _log.exception('cannot move hl7 file or log file to holding area')
     293          return True, target_names
     294
     295  #------------------------------------------------------------
     296 -def format_hl7_message(message=None, skip_empty_fields=True, eol='\n ', source=None):
     297          # a segment is a line starting with a type
     298          msg = pyhl7.parse(message)
     299
     300          output = []
     301          if source is not None:
     302                  output.append([_('HL7 Source'), '%s' % source])
     303          output.append([_('HL7 data size'), _('%s bytes') % len(message)])
     304          output.append([_('HL7 Message'), _(' %s segments (lines)%s') % (len(msg), gmTools.bool2subst(skip_empty_fields, _(', skipping empty fields'), ''))])
     305
     306          max_len = 0
     307          for seg_idx in range(len(msg)):
     308                  seg = msg[seg_idx]
     309                  seg_type = seg[0][0]
     310
     311                  output.append([_('Segment #%s <%s>') % (seg_idx, seg_type), _('%s fields') % len(seg)])
     312
     313                  for field_idx in range(len(seg)):
     314                          field = seg[field_idx]
     315                          try:
     316                                  label = HL7_field_labels[seg_type][field_idx]
     317                          except KeyError:
     318                                  label = _('HL7 %s field') % seg_type
     319
     320                          max_len = max(max_len, len(label))
     321
     322                          if len(field) == 0:
     323                                  if not skip_empty_fields:
     324                                          output.append(['%2s - %s' % (field_idx, label), _('<EMTPY>')])
     325                                  continue
     326                          if (len(field) == 1) and (('%s' % field[0]).strip() == ''):
     327                                  if not skip_empty_fields:
     328                                          output.append(['%2s - %s' % (field_idx, label), _('<EMTPY>')])
     329                                  continue
     330
     331                          content_lines = ('%s' % field).split(HL7_BRK)
     332                          output.append(['%2s - %s' % (field_idx, label), content_lines[0]])
     333                          for line in content_lines[1:]:
     334                                  output.append(['', line])
     335                          #output.append([u'%2s - %s' % (field_idx, label), u'%s' % field])
     336
     337          if eol is None:
     338                  return output
     339
     340          max_len += 7
     341          return eol.join([ '%s: %s' % ((o[0] + (' ' * max_len))[:max_len], o[1]) for o in output ])
     342
     343  #------------------------------------------------------------
     344 -def format_hl7_file(filename, skip_empty_fields=True, eol='\n ', return_filename=False, fix_hl7=True):
     345          if fix_hl7:
     346                  fixed_name = __fix_malformed_hl7_file(filename)
     347                  hl7_file = io.open(fixed_name, mode = 'rt', encoding = 'utf8', newline = '')    # read universal but pass on untranslated
     348                  source = '%s (<- %s)' % (fixed_name, filename)
     349          else:
     350                  hl7_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')      # read universal but pass on untranslated
     351                  source = filename
     352          output = format_hl7_message (
     353                  message = hl7_file.read(1024 * 1024 * 5),               # 5 MB max
     354                  skip_empty_fields = skip_empty_fields,
     355                  eol = eol,
     356                  source = source
     357          )
     358          hl7_file.close()
     359
     360          if not return_filename:
     361                  return output
     362
     363          if eol is None:
     364                  output = '\n '.join([ '%s: %s' % ((o[0] + (' ' * max_len))[:max_len], o[1]) for o in output ])
     365
     366          out_name = gmTools.get_unique_filename(prefix = 'gm-formatted_hl7-', suffix = '.hl7')
     367          out_file = io.open(out_name, mode = 'wt', encoding = 'utf8')
     368          out_file.write(output)
     369          out_file.close()
     370
     371          return out_name
     372
     373  #------------------------------------------------------------
     374  # this is used in the main code:
     375 -def stage_single_PID_hl7_file(filename, source=None, encoding='utf8'):
     376          """Multi-step processing of HL7 files.
     377
     378          - input must be single-MSH / single-PID / normalized HL7
     379
     380          - imports into clin.incoming_data_unmatched
     381
     382          - needs write permissions in dir_of(filename)
     383          - moves PID files which were successfully staged into dir_of(filename)/done/PID/
     384          """
     385          local_log_name = gmTools.get_unique_filename (
     386                  prefix = gmTools.fname_stem(filename) + '-',
     387                  suffix = '.stage.log'
     388          )
     389          local_logger = logging.FileHandler(local_log_name)
     390          local_logger.setLevel(logging.DEBUG)
     391          root_logger = logging.getLogger('')
     392          root_logger.addHandler(local_logger)
     393          _log.info('staging [%s] as unmatched incoming HL7%s', filename, gmTools.coalesce(source, '', ' (%s)'))
     394          _log.debug('log file: %s', local_log_name)
     395
     396          # sanity checks/setup
     397          try:
     398                  open(filename).close()
     399                  orig_dir = os.path.split(filename)[0]
     400                  done_dir = os.path.join(orig_dir, 'done')
     401                  gmTools.mkdir(done_dir)
     402                  error_dir = os.path.join(orig_dir, 'failed')
     403                  gmTools.mkdir(error_dir)
     404          except Exception:
     405                  _log.exception('cannot setup staging environment')
     406                  root_logger.removeHandler(local_logger)
     407                  return False
     408
     409          # stage
     410          try:
     411                  incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename)
     412                  if incoming is None:
     413                          _log.error('cannot stage PID file: %s', filename)
     414                          root_logger.removeHandler(local_logger)
     415                          shutil.move(filename, error_dir)
     416                          shutil.move(local_log_name, error_dir)
     417                          return False
     418                  incoming.update_data_from_file(fname = filename)
     419          except Exception:
     420                  _log.exception('error staging PID file')
     421                  root_logger.removeHandler(local_logger)
     422                  shutil.move(filename, error_dir)
     423                  shutil.move(local_log_name, error_dir)
     424                  return False
     425
     426          # set additional data
     427          MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
     428          raw_hl7 = MSH_file.read(1024 * 1024 * 5)        # 5 MB max
     429          MSH_file.close()
     430          shutil.move(filename, done_dir)
     431          incoming['comment'] = format_hl7_message (
     432                  message = raw_hl7,
     433                  skip_empty_fields = True,
     434                  eol = '\n'
     435          )
     436          HL7 = pyhl7.parse(raw_hl7)
     437          del raw_hl7
     438          incoming['comment'] += '\n'
     439          incoming['comment'] += ('-' * 80)
     440          incoming['comment'] += '\n\n'
     441          log = io.open(local_log_name, mode = 'rt', encoding = 'utf8')
     442          incoming['comment'] += log.read()
     443          log.close()
     444          try:
     445                  incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
     446                  incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
     447                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
     448                  if val is not None:
     449                          incoming['firstnames'] += ' '
     450                          incoming['firstnames'] += val
     451                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
     452                  if val is not None:
     453                          tmp = time.strptime(val, '%Y%m%d')
     454                          incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
     455                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)
     456                  if val is not None:
     457                          incoming['gender'] = val
     458                  incoming['external_data_id'] = filename
     459                  #u'fk_patient_candidates',
     460                  #       u'request_id',                                          # request ID as found in <data>
     461                  #       u'postcode',
     462                  #       u'other_info',                                          # other identifying info in .data
     463                  #       u'requestor',                                           # Requestor of data (e.g. who ordered test results) if available in source data.
     464                  #       u'fk_identity_disambiguated',
     465                  #       u'comment',                                                     # a free text comment on this row, eg. why is it here, error logs etc
     466                  #       u'fk_provider_disambiguated'            # The provider the data is relevant to.
     467          except Exception:
     468                  _log.exception('cannot add more data')
     469          incoming.save()
     470
     471          _log.info('successfully staged')
     472          root_logger.removeHandler(local_logger)
     473          shutil.move(local_log_name, done_dir)
     474          return True
     475
     476  #------------------------------------------------------------
     477 -def process_staged_single_PID_hl7_file(staged_item):
     478
     479          log_name = gmTools.get_unique_filename (
     480                  prefix = 'gm-staged_hl7_import-',
     481                  suffix = '.log'
     482          )
     483          import_logger = logging.FileHandler(log_name)
     484          import_logger.setLevel(logging.DEBUG)
     485          root_logger = logging.getLogger('')
     486          root_logger.addHandler(import_logger)
     487          _log.debug('log file: %s', log_name)
     488
     489          if not staged_item.lock():
     490                  _log.error('cannot lock staged data for HL7 import')
     491                  root_logger.removeHandler(import_logger)
     492                  return False, log_name
     493
     494          _log.debug('reference ID of staged HL7 data: %s', staged_item['external_data_id'])
     495
     496          filename = staged_item.save_to_file()
     497          _log.debug('unstaged HL7 data into: %s', filename)
     498
     499          if staged_item['pk_identity_disambiguated'] is None:
     500                  emr = None
     501          else:
     502                  emr = gmPerson.cPatient(staged_item['pk_identity_disambiguated']).emr
     503
     504          success = False
     505          try:
     506                  success = __import_single_PID_hl7_file(filename, emr = emr)
     507                  if success:
     508                          gmIncomingData.delete_incoming_data(pk_incoming_data = staged_item['pk_incoming_data_unmatched'])
     509                          staged_item.unlock()
     510                          root_logger.removeHandler(import_logger)
     511                          return True, log_name
     512                  _log.error('error when importing single-PID/single-MSH file')
     513          except Exception:
     514                  _log.exception('error when importing single-PID/single-MSH file')
     515
     516          if not success:
     517                  staged_item['comment'] = _('failed import: %s\n') % gmDateTime.pydt_strftime(gmDateTime.pydt_now_here())
     518                  staged_item['comment'] += '\n'
     519                  staged_item['comment'] += ('-' * 80)
     520                  staged_item['comment'] += '\n\n'
     521                  log = io.open(log_name, mode = 'rt', encoding = 'utf8')
     522                  staged_item['comment'] += log.read()
     523                  log.close()
     524                  staged_item['comment'] += '\n'
     525                  staged_item['comment'] += ('-' * 80)
     526                  staged_item['comment'] += '\n\n'
     527                  staged_item['comment'] += format_hl7_file (
     528                          filename,
     529                          skip_empty_fields = True,
     530                          eol = '\n ',
     531                          return_filename = False
     532                  )
     533                  staged_item.save()
     534
     535          staged_item.unlock()
     536          root_logger.removeHandler(import_logger)
     537          return success, log_name
     538
     539  #------------------------------------------------------------
     540 -def import_single_PID_hl7_file(filename):
     541
     542          log_name = '%s.import.log' % filename
     543          import_logger = logging.FileHandler(log_name)
     544          import_logger.setLevel(logging.DEBUG)
     545          root_logger = logging.getLogger('')
     546          root_logger.addHandler(import_logger)
     547          _log.debug('log file: %s', log_name)
     548
     549          success = True
     550          try:
     551                  success = __import_single_PID_hl7_file(filename)
     552                  if not success:
     553                          _log.error('error when importing single-PID/single-MSH file')
     554          except Exception:
     555                  _log.exception('error when importing single-PID/single-MSH file')
     556
     557          root_logger.removeHandler(import_logger)
     558          return success, log_name
     559
     560  #============================================================
     561  # internal helpers
     562  #============================================================
     563 -def __fix_malformed_hl7_file(filename, encoding='utf8'):
     564
     565          _log.debug('fixing HL7 file [%s]', filename)
     566
     567          # first pass:
     568          # - remove empty lines
     569          # - normalize line endings
     570          # - unwrap wrapped segments (based on the assumption that segments are wrapped until a line starts with a known segment marker)
     571          out1_fname = gmTools.get_unique_filename (
     572                  prefix = 'gm_fix1-%s-' % gmTools.fname_stem(filename),
     573                  suffix = '.hl7'
     574          )
     575          hl7_in = io.open(filename, mode = 'rt', encoding = encoding)                                    # universal newlines: translate any type of EOL to \n
     576          hl7_out = io.open(out1_fname, mode = 'wt', encoding = 'utf8', newline = '')     # newline='' -> no translation of EOL at all
     577          is_first_line = True
     578          for line in hl7_in:
     579                  # skip empty line
     580                  if line.strip() == '':
     581                          continue
     582                  # starts with known segment ?
     583                  segment = line[:3]
     584                  if (segment in HL7_SEGMENTS) and (line[3] == '|'):
     585                          if not is_first_line:
     586                                  hl7_out.write(HL7_EOL)
     587                          else:
     588                                  is_first_line = False
     589                  else:
     590                          hl7_out.write(' ')
     591                  hl7_out.write(line.rstrip())
     592          hl7_out.write(HL7_EOL)
     593          hl7_out.close()
     594          hl7_in.close()
     595
     596          # second pass:
     597          # - normalize # of fields per line
     598          # - remove '\.br.\'-only fields ;-)
     599          out2_fname = gmTools.get_unique_filename (
     600                  prefix = 'gm_fix2-%s-' % gmTools.fname_stem(filename),
     601                  suffix = '.hl7'
     602          )
     603          # we can now _expect_ lines to end in HL7_EOL, anything else is an error
     604          hl7_in = io.open(out1_fname, mode = 'rt', encoding = 'utf8', newline = HL7_EOL)
     605          hl7_out = io.open(out2_fname, mode = 'wt', encoding = 'utf8', newline = '')
     606          for line in hl7_in:
     607                  line = line.strip()
     608                  seg_type = line[:3]                                             # assumption: field separator = '|'
     609                  field_count = line.count('|') + 1               # assumption: no '|' in data ...
     610                  try:
     611                          required_fields = HL7_segment2field_count[seg_type]
     612                  except KeyError:
     613                          required_fields = field_count
     614                  missing_fields_count = required_fields - field_count
     615                  if missing_fields_count > 0:
     616                          line += ('|' * missing_fields_count)
     617                  cleaned_fields = []
     618                  for field in line.split('|'):
     619                          if field.replace(HL7_BRK, '').strip() == '':
     620                                  cleaned_fields.append('')
     621                                  continue
     622                          cleaned = gmTools.strip_prefix(field, HL7_BRK, remove_repeats = True, remove_whitespace = True)
     623                          cleaned = gmTools.strip_suffix(cleaned, HL7_BRK, remove_repeats = True, remove_whitespace = True)
     624                          cleaned_fields.append(cleaned)
     625                  hl7_out.write('|'.join(cleaned_fields) + HL7_EOL)
     626          hl7_out.close()
     627          hl7_in.close()
     628
     629          # third pass:
     630          # - unsplit same-name, same-time, text-type OBX segments
     631          out3_fname = gmTools.get_unique_filename (
     632                  prefix = 'gm_fix3-%s-' % gmTools.fname_stem(filename),
     633                  suffix = '.hl7'
     634          )
     635          # we can now _expect_ lines to end in HL7_EOL, anything else is an error
     636          hl7_in = io.open(out2_fname, mode = 'rt', encoding = 'utf8', newline = HL7_EOL)
     637          hl7_out = io.open(out3_fname, mode = 'wt', encoding = 'utf8', newline = '')
     638          prev_identity = None
     639          prev_fields = None
     640          for line in hl7_in:
     641                  line = line.strip()
     642                  if not line.startswith('OBX|'):
     643                          if prev_fields is not None:
     644                                  hl7_out.write('|'.join(prev_fields) + HL7_EOL)
     645                          hl7_out.write(line + HL7_EOL)
     646                          prev_identity = None
     647                          prev_fields = None
     648                          curr_fields = None
     649                          continue
     650                  # first OBX
     651                  curr_fields = line.split('|')
     652                  if curr_fields[OBX_field__datatype] != 'FT':
     653                          hl7_out.write(line + HL7_EOL)
     654                          prev_identity = None
     655                          prev_fields = None
     656                          curr_fields = None
     657                          continue
     658                  # first FT type OBX
     659                  if prev_fields is None:
     660                          prev_fields = line.split('|')
     661                          prev_identity = line.split('|')
     662                          prev_identity[OBX_field__set_id] = ''
     663                          prev_identity[OBX_field__subid] = ''
     664                          prev_identity[OBX_field__value] = ''
     665                          prev_identity = '|'.join(prev_identity)
     666                          continue
     667                  # non-first FT type OBX
     668                  curr_identity = line.split('|')
     669                  curr_identity[OBX_field__set_id] = ''
     670                  curr_identity[OBX_field__subid] = ''
     671                  curr_identity[OBX_field__value] = ''
     672                  curr_identity = '|'.join(curr_identity)
     673                  if curr_identity != prev_identity:
     674                          # write out previous line
     675                          hl7_out.write('|'.join(prev_fields) + HL7_EOL)
     676                          # keep current fields, since it may start a "repeat FT type OBX block"
     677                          prev_fields = curr_fields
     678                          prev_identity = curr_identity
     679                          continue
     680                  if prev_fields[OBX_field__value].endswith(HL7_BRK):
     681                          prev_fields[OBX_field__value] += curr_fields[OBX_field__value]
     682                  else:
     683                          if curr_fields[OBX_field__value].startswith(HL7_BRK):
     684                                  prev_fields[OBX_field__value] += curr_fields[OBX_field__value]
     685                          else:
     686                                  prev_fields[OBX_field__value] += HL7_BRK
     687                                  prev_fields[OBX_field__value] += curr_fields[OBX_field__value]
     688          if prev_fields is not None:
     689                  hl7_out.write('|'.join(prev_fields) + HL7_EOL)
     690          hl7_out.close()
     691          hl7_in.close()
     692
     693          return out3_fname
     694
     695  #------------------------------------------------------------
     696 -def __split_hl7_file_by_MSH(filename, encoding='utf8'):
     697
     698          _log.debug('splitting [%s] into single-MSH files', filename)
     699
     700          hl7_in = io.open(filename, mode = 'rt', encoding = encoding)
     701
     702          idx = 0
     703          first_line = True
     704          MSH_file = None
     705          MSH_fnames = []
     706          for line in hl7_in:
     707                  line = line.strip()
     708                  # first line must be MSH
     709                  if first_line:
     710                          # ignore empty / FHS / BHS lines
     711                          if line == '':
     712                                  continue
     713                          if line.startswith('FHS|'):
     714                                  _log.debug('ignoring FHS')
     715                                  continue
     716                          if line.startswith('BHS|'):
     717                                  _log.debug('ignoring BHS')
     718                                  continue
     719                          if not line.startswith('MSH|'):
     720                                  raise ValueError('HL7 file <%s> does not start with "MSH" line' % filename)
     721                          first_line = False
     722                  # start new file
     723                  if line.startswith('MSH|'):
     724                          if MSH_file is not None:
     725                                  MSH_file.close()
     726                          idx += 1
     727                          out_fname = gmTools.get_unique_filename(prefix = '%s-MSH_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7')
     728                          _log.debug('writing message %s to [%s]', idx, out_fname)
     729                          MSH_fnames.append(out_fname)
     730                          MSH_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '')
     731                  # ignore BTS / FTS lines
     732                  if line.startswith('BTS|'):
     733                          _log.debug('ignoring BTS')
     734                          continue
     735                  if line.startswith('FTS|'):
     736                          _log.debug('ignoring FTS')
     737                          continue
     738                  # else write line to new file
     739                  MSH_file.write(line + HL7_EOL)
     740
     741          if MSH_file is not None:
     742                  MSH_file.close()
     743          hl7_in.close()
     744
     745          return MSH_fnames
     746
     747  #------------------------------------------------------------
     748 -def __split_MSH_by_PID(filename):
     749          """Assumes:
     750                  - ONE MSH per file
     751                  - utf8 encoding
     752                  - first non-empty line must be MSH line
     753                  - next line must be PID line
     754
     755                  IOW, what's created by __split_hl7_file_by_MSH()
     756          """
     757          _log.debug('splitting single-MSH file [%s] into single-PID files', filename)
     758
     759          MSH_in = io.open(filename, mode = 'rt', encoding = 'utf8')
     760
     761          looking_for_MSH = True
     762          MSH_line = None
     763          looking_for_first_PID = True
     764          PID_file = None
     765          PID_fnames = []
     766          idx = 0
     767          for line in MSH_in:
     768                  line = line.strip()
     769                  # ignore empty
     770                  if line == '':
     771                          continue
     772
     773                  # first non-empty line must be MSH
     774                  if looking_for_MSH:
     775                          if line.startswith('MSH|'):
     776                                  looking_for_MSH = False
     777                                  MSH_line = line + HL7_EOL
     778                                  continue
     779                          raise ValueError('HL7 MSH file <%s> does not start with "MSH" line' % filename)
     780                  else:
     781                          if line.startswith('MSH|'):
     782                                  raise ValueError('HL7 single-MSH file <%s> contains more than one MSH line' % filename)
     783
     784                  # first non-empty line after MSH must be PID
     785                  if looking_for_first_PID:
     786                          if not line.startswith('PID|'):
     787                                  raise ValueError('HL7 MSH file <%s> does not have "PID" line follow "MSH" line' % filename)
     788                          looking_for_first_PID = False
     789
     790                  # start new file if line is PID
     791                  if line.startswith('PID|'):
     792                          if PID_file is not None:
     793                                  PID_file.close()
     794                          idx += 1
     795                          out_fname = gmTools.get_unique_filename(prefix = '%s-PID_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7')
     796                          _log.debug('writing message for PID %s to [%s]', idx, out_fname)
     797                          PID_fnames.append(out_fname)
     798                          PID_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '')
     799                          PID_file.write(MSH_line)
     800                  # else write line to new file
     801                  PID_file.write(line + HL7_EOL)
     802
     803          if PID_file is not None:
     804                  PID_file.close()
     805          MSH_in.close()
     806
     807          return PID_fnames
     808
     809  #------------------------------------------------------------
     810 -def __find_or_create_lab(hl7_lab, link_obj=None):
     811          comment_tag = '[HL7 name::%s]' % hl7_lab
     812          for gm_lab in gmPathLab.get_test_orgs():
     813                  if comment_tag in gmTools.coalesce(gm_lab['comment'], ''):
     814                          _log.debug('found lab [%s] from HL7 file in GNUmed database:', hl7_lab)
     815                          _log.debug(gm_lab)
     816                          return gm_lab
     817          _log.debug('lab not found: %s', hl7_lab)
     818          gm_lab = gmPathLab.create_test_org(link_obj = link_obj, name = hl7_lab, comment = comment_tag)
     819          if gm_lab is None:
     820                  raise ValueError('cannot create lab [%s] in GNUmed' % hl7_lab)
     821          _log.debug('created lab: %s', gm_lab)
     822          return gm_lab
     823
     824  #------------------------------------------------------------
     825 -def __find_or_create_test_type(loinc=None, name=None, pk_lab=None, unit=None, link_obj=None, abbrev=None):
     826
     827          tt = gmPathLab.find_measurement_type(link_obj = link_obj, lab = pk_lab, name = name)
     828          if tt is None:
     829                  _log.debug('test type [%s::%s::%s] not found for lab #%s, creating', name, unit, loinc, pk_lab)
     830                  tt = gmPathLab.create_measurement_type(link_obj = link_obj, lab = pk_lab, abbrev = gmTools.coalesce(abbrev, name), unit = unit, name = name)
     831                  _log.debug('created as: %s', tt)
     832
     833          if loinc is None:
     834                  return tt
     835          if loinc.strip() == '':
     836                  return tt
     837          if tt['loinc'] is None:
     838                  tt['loinc'] = loinc
     839                  tt.save(conn = link_obj)
     840                  return tt
     841          if tt['loinc'] != loinc:
     842  #               raise ValueError('LOINC code mismatch between GM (%s) and HL7 (%s) for result type [%s]' % (tt['loinc'], loinc, name))
     843                  _log.error('LOINC code mismatch between GM (%s) and HL7 (%s) for result type [%s]', tt['loinc'], loinc, name)
     844
     845          return tt
     846
     847  #------------------------------------------------------------
     848 -def __ensure_hl7_test_types_exist_in_gnumed(link_obj=None, hl7_data=None, pk_test_org=None):
     849          try:
     850                  OBX_count = len(hl7_data.segments('OBX'))
     851          except KeyError:
     852                  _log.error("HL7 does not contain OBX segments, nothing to do")
     853                  return
     854          for OBX_idx in range(OBX_count):
     855                  unit = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__unit)
     856                  if unit == '':
     857                          unit = None
     858                  LOINC = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__loinc)
     859                  tname = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__name)
     860                  tt = __find_or_create_test_type (
     861                          loinc = LOINC,
     862                          name = tname,
     863                          pk_lab = pk_test_org,
     864                          unit = unit,
     865                          link_obj = link_obj
     866                  )
     867
     868  #------------------------------------------------------------
     869 -def __PID2dto(HL7=None):
     870          pat_lname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
     871          pat_fname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
     872          pat_mname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
     873          if pat_mname is not None:
     874                  pat_fname += ' '
     875                  pat_fname += pat_mname
     876          _log.debug('patient data from PID segment: first=%s (middle=%s) last=%s', pat_fname, pat_mname, pat_lname)
     877
     878          dto = gmPerson.cDTO_person()
     879          dto.firstnames = pat_fname
     880          dto.lastnames = pat_lname
     881          dto.gender = HL7_GENDERS[HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)]
     882          hl7_dob = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
     883          if hl7_dob is not None:
     884                  tmp = time.strptime(hl7_dob, '%Y%m%d')
     885                  dto.dob = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
     886
     887          idents = dto.get_candidate_identities()
     888          if len(idents) == 0:
     889                  _log.warning('no match candidate, not auto-importing')
     890                  _log.debug(dto)
     891                  return []
     892          if len(idents) > 1:
     893                  _log.warning('more than one match candidate, not auto-importing')
     894                  _log.debug(dto)
     895                  return idents
     896          return [gmPerson.cPatient(idents[0].ID)]
     897
     898  #------------------------------------------------------------
     899 -def __hl7dt2pydt(hl7dt):
     900          if hl7dt == '':
     901                  return None
     902
     903          if len(hl7dt) == 8:
     904                  tmp = time.strptime(hl7dt, '%Y%m%d')
     905                  return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
     906
     907          if len(hl7dt) == 12:
     908                  tmp = time.strptime(hl7dt, '%Y%m%d%H%M')
     909                  return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tzinfo = gmDateTime.gmCurrentLocalTimezone)
     910
     911          if len(hl7dt) == 14:
     912                  tmp = time.strptime(hl7dt, '%Y%m%d%H%M%S')
     913                  return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tmp.tm_sec, tzinfo = gmDateTime.gmCurrentLocalTimezone)
     914
     915          raise ValueError('Observation timestamp not parseable: [%s]', hl7dt)
     916
     917  #------------------------------------------------------------
     918 -def __import_single_PID_hl7_file(filename, emr=None):
     919          """Assumes single-PID/single-MSH HL7 file."""
     920
     921          _log.debug('importing single-PID single-MSH HL7 data from [%s]', filename)
     922
     923          # read the file
     924          MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
     925          HL7 = pyhl7.parse(MSH_file.read(1024 * 1024 * 5))       # 5 MB max
     926          MSH_file.close()
     927
     928          # sanity checks
     929          if len(HL7.segments('MSH')) != 1:
     930                  _log.error('more than one MSH segment')
     931                  return False
     932          if len(HL7.segments('PID')) != 1:
     933                  _log.error('more than one PID segment')
     934                  return False
     935
     936          # ensure lab is in database
     937          hl7_lab = HL7.extract_field('MSH', field_num = MSH_field__sending_lab)
     938          gm_lab = __find_or_create_lab(hl7_lab)
     939
     940          # ensure test types exist
     941          conn = gmPG2.get_connection(readonly = False)
     942          __ensure_hl7_test_types_exist_in_gnumed(link_obj = conn, hl7_data = HL7, pk_test_org = gm_lab['pk_test_org'])
     943
     944          # find patient
     945          if emr is None:
     946                  #PID = HL7.segment('PID')
     947                  pats = __PID2dto(HL7 = HL7)
     948                  if len(pats) == 0:
     949                          conn.rollback()
     950                          return False
     951                  if len(pats) > 1:
     952                          conn.rollback()
     953                          return False
     954                  emr = pats[0].emr
     955
     956          # import values: loop over segments
     957          when_list = {}
     958          current_result = None
     959          previous_segment = None
     960          had_errors = False
     961          msh_seen = False
     962          pid_seen = False
     963          last_obr = None
     964          obr = {}
     965          for seg_idx in range(len(HL7)):
     966                  seg = HL7[seg_idx]
     967                  seg_type = seg[0][0]
     968
     969                  _log.debug('processing line #%s = segment of type <%s>', seg_idx, seg_type)
     970
     971                  if seg_type == 'MSH':
     972                          msh_seen = True
     973
     974                  if seg_type == 'PID':
     975                          if not msh_seen:
     976                                  conn.rollback()
     977                                  _log.error('PID segment before MSH segment')
     978                                  return False
     979                          pid_seen = True
     980
     981                  if seg_type in ['MSH', 'PID']:
     982                          _log.info('segment already handled')
     983                          previous_segment = seg_type
     984                          obr = {}
     985                          current_result = None
     986                          continue
     987
     988                  if seg_type in ['ORC']:
     989                          _log.info('currently ignoring %s segments', seg_type)
     990                          previous_segment = seg_type
     991                          obr = {}
     992                          current_result = None
     993                          continue
     994
     995                  if seg_type == 'OBR':
     996                          previous_segment = seg_type
     997                          last_obr = seg
     998                          current_result = None
     999                          obr['abbrev'] = ('%s' % seg[OBR_field__service_name][0]).strip()
    1000                          try:
    1001                                  obr['name'] = ('%s' % seg[OBR_field__service_name][1]).strip()
    1002                          except IndexError:
    1003                                  obr['name'] = obr['abbrev']
    1004                          for field_name in [OBR_field__ts_ended, OBR_field__ts_started, OBR_field__ts_specimen_received, OBR_field__ts_requested]:
    1005                                  obr['clin_when'] = seg[field_name][0].strip()
    1006                                  if obr['clin_when'] != '':
    1007                                          break
    1008                          continue
    1009
    1010                  if seg_type == 'OBX':
    1011                          current_result = None
    1012                          # determine value
    1013                          val_alpha = seg[OBX_field__value][0].strip()
    1014                          is_num, val_num = gmTools.input2decimal(initial = val_alpha)
    1015                          if is_num:
    1016                                  val_alpha = None
    1017                          else:
    1018                                  val_num = None
    1019                                  val_alpha = val_alpha.replace('\.br\\', '\n')
    1020                          # determine test type
    1021                          unit = seg[OBX_field__unit][0].strip()
    1022                          if unit == '':
    1023                                  if is_num:
    1024                                          unit = '1/1'
    1025                                  else:
    1026                                          unit = None
    1027                          test_type = __find_or_create_test_type (
    1028                                  loinc = '%s' % seg[OBX_field__type][0][OBX_component__loinc-1],
    1029                                  name = '%s' % seg[OBX_field__type][0][OBX_component__name-1],
    1030                                  pk_lab = gm_lab['pk_test_org'],
    1031                                  unit = unit,
    1032                                  link_obj = conn
    1033                          )
    1034                          # eventually, episode should be read from lab_request
    1035                          epi = emr.add_episode (
    1036                                  link_obj = conn,
    1037                                  episode_name = 'administrative',
    1038                                  is_open = False,
    1039                                  allow_dupes = False
    1040                          )
    1041                          current_result = emr.add_test_result (
    1042                                  link_obj = conn,
    1043                                  episode = epi['pk_episode'],
    1044                                  type = test_type['pk_test_type'],
    1045                                  intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'],
    1046                                  val_num = val_num,
    1047                                  val_alpha = val_alpha,
    1048                                  unit = unit
    1049                          )
    1050                          # handle range information et al
    1051                          ref_range = seg[OBX_field__range][0].strip()
    1052                          if ref_range != '':
    1053                                  current_result.reference_range = ref_range
    1054                          flag = seg[OBX_field__abnormal_flag][0].strip()
    1055                          if flag != '':
    1056                                  current_result['abnormality_indicator'] = flag
    1057                          current_result['status'] = seg[OBX_field__status][0].strip()
    1058                          current_result['val_grouping'] = seg[OBX_field__subid][0].strip()
    1059                          current_result['source_data'] = ''
    1060                          if last_obr is not None:
    1061                                  current_result['source_data'] += str(last_obr)
    1062                                  current_result['source_data'] += '\n'
    1063                          current_result['source_data'] += str(seg)
    1064                          clin_when = seg[OBX_field__timestamp][0].strip()
    1065                          if clin_when == '':
    1066                                  _log.warning('no <Observation timestamp> in OBX, trying OBR timestamp')
    1067                                  clin_when = obr['clin_when']
    1068                          try:
    1069                                  clin_when = __hl7dt2pydt(clin_when)
    1070                          except ValueError:
    1071                                  _log.exception('clin_when from OBX or OBR not useable, assuming <today>')
    1072                          if clin_when is not None:
    1073                                  current_result['clin_when'] = clin_when
    1074                          current_result.save(conn = conn)
    1075                          when_list[gmDateTime.pydt_strftime(current_result['clin_when'], '%Y %b %d')] = 1
    1076                          previous_segment = seg_type
    1077                          continue
    1078
    1079                  if seg_type == 'NTE':
    1080                          note = seg[NET_field__note][0].strip().replace('\.br\\', '\n')
    1081                          if note == '':
    1082                                  _log.debug('empty NTE segment')
    1083                                  previous_segment = seg_type                     # maybe not ? (HL7 providers happen to use empty NTE segments to "structure" raw HL7 |-)
    1084                                  continue
    1085
    1086                          # if this is an NTE following an OBR (IOW an order-related
    1087                          # comment): make this a test result all of its own :-)
    1088                          if previous_segment == 'OBR':
    1089                                  _log.debug('NTE following OBR: general note, using OBR timestamp [%s]', obr['clin_when'])
    1090                                  current_result = None
    1091                                  name = obr['name']
    1092                                  if name == '':
    1093                                          name = _('Comment')
    1094                                  # FIXME: please suggest a LOINC for "order comment"
    1095                                  test_type = __find_or_create_test_type(name = name, pk_lab = gm_lab['pk_test_org'], abbrev = obr['abbrev'], link_obj = conn)
    1096                                  # eventually, episode should be read from lab_request
    1097                                  epi = emr.add_episode (
    1098                                          link_obj = conn,
    1099                                          episode_name = 'administrative',
    1100                                          is_open = False,
    1101                                          allow_dupes = False
    1102                                  )
    1103                                  nte_result = emr.add_test_result (
    1104                                          link_obj = conn,
    1105                                          episode = epi['pk_episode'],
    1106                                          type = test_type['pk_test_type'],
    1107                                          intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'],
    1108                                          val_alpha = note
    1109                                  )
    1110                                  #nte_result['val_grouping'] = seg[OBX_field__subid][0].strip()
    1111                                  nte_result['source_data'] = str(seg)
    1112                                  try:
    1113                                          nte_result['clin_when'] = __hl7dt2pydt(obr['clin_when'])
    1114                                  except ValueError:
    1115                                          _log.exception('no .clin_when from OBR for NTE pseudo-OBX available')
    1116                                  nte_result.save(conn = conn)
    1117                                  continue
    1118
    1119                          if (previous_segment == 'OBX') and (current_result is not None):
    1120                                  current_result['source_data'] += '\n'
    1121                                  current_result['source_data'] += str(seg)
    1122                                  current_result['note_test_org'] = gmTools.coalesce (
    1123                                          current_result['note_test_org'],
    1124                                          note,
    1125                                          '%%s\n%s' % note
    1126                                  )
    1127                                  current_result.save(conn = conn)
    1128                                  previous_segment = seg_type
    1129                                  continue
    1130
    1131                          _log.error('unexpected NTE segment')
    1132                          had_errors = True
    1133                          break
    1134
    1135                  _log.error('unknown segment, aborting')
    1136                  _log.debug('line: %s', seg)
    1137                  had_errors = True
    1138                  break
    1139
    1140          if had_errors:
    1141                  conn.rollback()
    1142                  return False
    1143
    1144          conn.commit()
    1145
    1146          # record import in chart
    1147          try:
    1148                  no_results = len(HL7.segments('OBX'))
    1149          except KeyError:
    1150                  no_results = '?'
    1151          soap = _(
    1152                  'Imported HL7 file [%s]:\n'
    1153                  ' lab "%s" (%s@%s), %s results (%s)'
    1154          ) % (
    1155                  filename,
    1156                  hl7_lab,
    1157                  gm_lab['unit'],
    1158                  gm_lab['organization'],
    1159                  no_results,
    1160                  ' / '.join(when_list.keys())
    1161          )
    1162          epi = emr.add_episode (
    1163                  episode_name = 'administrative',
    1164                  is_open = False,
    1165                  allow_dupes = False
    1166          )
    1167          emr.add_clin_narrative (
    1168                  note = soap,
    1169                  soap_cat = None,
    1170                  episode = epi
    1171          )
    1172
    1173          # keep copy of HL7 data in document archive
    1174          folder = gmPerson.cPatient(emr.pk_patient).document_folder
    1175          hl7_docs = folder.get_documents (
    1176                  doc_type = 'HL7 data',
    1177                  pk_episodes = [epi['pk_episode']],
    1178                  order_by = 'ORDER BY clin_when DESC'
    1179          )
    1180          if len(hl7_docs) > 0:
    1181                  # there should only ever be one unless the user manually creates more,
    1182                  # also, it should always be the latest since "ORDER BY clin_when DESC"
    1183                  hl7_doc = hl7_docs[0]
    1184          else:
    1185                  hl7_doc = folder.add_document (
    1186                          document_type = 'HL7 data',
    1187                          encounter = emr.active_encounter['pk_encounter'],
    1188                          episode = epi['pk_episode']
    1189                  )
    1190                  hl7_doc['comment'] = _('list of imported HL7 data files')
    1191                  hl7_doc['pk_org_unit'] = gmPraxis.gmCurrentPraxisBranch()['pk_org_unit']
    1192          hl7_doc['clin_when'] = gmDateTime.pydt_now_here()
    1193          hl7_doc.save()
    1194          part = hl7_doc.add_part(file = filename)
    1195          part['obj_comment'] = _('Result dates: %s') % ' / '.join(when_list.keys())
    1196          part.save()
    1197          hl7_doc.set_reviewed(technically_abnormal = False, clinically_relevant = False)
    1198
    1199          return True
    1200
    1201  #------------------------------------------------------------
    1202  # this is only used for testing here in this file
    1203 -def __stage_MSH_as_incoming_data(filename, source=None, logfile=None):
    1204          """Consumes single-MSH single-PID HL7 files."""
    1205
    1206          _log.debug('staging [%s] as unmatched incoming HL7%s', gmTools.coalesce(source, '', ' (%s)'), filename)
    1207
    1208          # parse HL7
    1209          MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
    1210          raw_hl7 = MSH_file.read(1024 * 1024 * 5)        # 5 MB max
    1211          MSH_file.close()
    1212          formatted_hl7 = format_hl7_message (
    1213                  message = raw_hl7,
    1214                  skip_empty_fields = True,
    1215                  eol = '\n'
    1216          )
    1217          HL7 = pyhl7.parse(raw_hl7)
    1218          del raw_hl7
    1219
    1220          # import file
    1221          incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename)
    1222          if incoming is None:
    1223                  return None
    1224          incoming.update_data_from_file(fname = filename)
    1225          incoming['comment'] = formatted_hl7
    1226          if logfile is not None:
    1227                  log = io.open(logfile, mode = 'rt', encoding = 'utf8')
    1228                  incoming['comment'] += '\n'
    1229                  incoming['comment'] += ('-' * 80)
    1230                  incoming['comment'] += '\n\n'
    1231                  incoming['comment'] += log.read()
    1232                  log.close()
    1233          try:
    1234                  incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
    1235                  incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
    1236                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
    1237                  if val is not None:
    1238                          incoming['firstnames'] += ' '
    1239                          incoming['firstnames'] += val
    1240                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
    1241                  if val is not None:
    1242                          tmp = time.strptime(val, '%Y%m%d')
    1243                          incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
    1244                  val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)
    1245                  if val is not None:
    1246                          incoming['gender'] = val
    1247                  incoming['external_data_id'] = filename
    1248                  #u'fk_patient_candidates',
    1249                  #       u'request_id',                                          # request ID as found in <data>
    1250                  #       u'postcode',
    1251                  #       u'other_info',                                          # other identifying info in .data
    1252                  #       u'requestor',                                           # Requestor of data (e.g. who ordered test results) if available in source data.
    1253                  #       u'fk_identity_disambiguated',
    1254                  #       u'comment',                                                     # a free text comment on this row, eg. why is it here, error logs etc
    1255                  #       u'fk_provider_disambiguated'            # The provider the data is relevant to.
    1256          except KeyError:
    1257                  _log.exception('no PID segment, cannot add more data')
    1258          incoming.save()
    1259
    1260          return incoming
    1261
    1262  #============================================================
    1263  # main
    1264  #------------------------------------------------------------
    1265  if __name__ == "__main__":
    1266
    1267          if len(sys.argv) < 2:
    1268                  sys.exit()
    1269
    1270          if sys.argv[1] != 'test':
    1271                  sys.exit()
    1272
    1273          from Gnumed.pycommon import gmLog2
    1274
    1275          gmDateTime.init()
    1276          gmTools.gmPaths()
    1277
    1278          #-------------------------------------------------------
    1279 -        def test_import_HL7(filename):
    1280                  # would normally be set by external configuration:
    1281                  from Gnumed.business import gmPraxis
    1282                  gmPraxis.gmCurrentPraxisBranch(branch = gmPraxis.get_praxis_branches()[0])
    1283                  if not import_hl7_file(filename):
    1284                          print("error with", filename)
    1285          #-------------------------------------------------------
    1286 -        def test_xml_extract():
    1287                  hl7 = extract_HL7_from_XML_CDATA(sys.argv[2], './/Message')
    1288                  print("HL7:", hl7)
    1289                  #result, PID_fnames = split_hl7_file(hl7)
    1290                  #print "result:", result
    1291                  #print "per-PID MSH files:"
    1292                  #for name in PID_fnames:
    1293                  #       print " ", name
    1294          #-------------------------------------------------------
    1295 -        def test_stage_hl7_from_xml():
    1296                  hl7 = extract_HL7_from_XML_CDATA(sys.argv[2], './/Message')
    1297                  print("HL7:", hl7)
    1298                  result, PID_fnames = split_hl7_file(hl7)
    1299                  print("result:", result)
    1300                  print("staging per-PID HL7 files:")
    1301                  for name in PID_fnames:
    1302                          print(" file:", name)
    1303                          __stage_MSH_as_incoming_data(name, source = 'Excelleris')
    1304          #-------------------------------------------------------
    1305 -        def test_split_hl7_file():
    1306                  result, PID_fnames = split_hl7_file(sys.argv[2])
    1307                  print("result:", result)
    1308                  print("per-PID HL7 files:")
    1309                  for name in PID_fnames:
    1310                          print(" file:", name)
    1311          #-------------------------------------------------------
    1312 -        def test_stage_hl7():
    1313                  fixed = __fix_malformed_hl7_file(sys.argv[2])
    1314                  print("fixed HL7:", fixed)
    1315                  PID_fnames = split_HL7_by_PID(fixed, encoding='utf8')
    1316                  print("staging per-PID HL7 files:")
    1317                  for name in PID_fnames:
    1318                          print(" file:", name)
    1319                          #print "", __stage_MSH_as_incoming_data(name, source = u'?')
    1320          #-------------------------------------------------------
    1321 -        def test_format_hl7_message():
    1322                  tests = [
    1323                          "OBR|1||03-1350023-LIP-0|LIP^Lipids||20031004073300|20031004073300|||||||20031004073300||22333^MEDIC^IAN^TEST||031350023||03-1350023|031350023|20031004131600||CHEM|F|||22333^MEDIC^IAN^TEST",
    1324                          "OBX|2|NM|22748-8^LDL Cholesterol||4.0|mmol/L|1.5 - 3.4|H|||F|||20031004073300"
    1325                  ]
    1326                  for test in tests:
    1327                          print(format_hl7_message (
    1328  #                               skip_empty_fields = True,
    1329                                  message = test
    1330                          ))
    1331          #-------------------------------------------------------
    1332 -        def test_format_hl7_file(filename):
    1333                  print(format_hl7_file (
    1334                          filename,
    1335  #                       skip_empty_fields = True
    1336                          return_filename = True
    1337                  ))
    1338          #-------------------------------------------------------
    1339 -        def test___fix_malformed_hl7():
    1340                  print("fixed HL7:", __fix_malformed_hl7_file(sys.argv[2]))
    1341          #-------------------------------------------------------
    1342 -        def test_parse_hl7():
    1343                  MSH_file = io.open(sys.argv[2], mode = 'rt', encoding = 'utf8', newline = '')
    1344                  raw_hl7 = MSH_file.read(1024 * 1024 * 5)        # 5 MB max
    1345                  MSH_file.close()
    1346                  print(format_hl7_message (
    1347                          message = raw_hl7,
    1348                          skip_empty_fields = True,
    1349                          eol = '\n'
    1350                  ))
    1351                  HL7 = pyhl7.parse(raw_hl7)
    1352                  del raw_hl7
    1353                  for seg in HL7.segments('MSH'):
    1354                          print(seg)
    1355                  print("PID:")
    1356                  print(HL7.extract_field('PID'))
    1357                  print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname))
    1358                  print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname))
    1359
    1360  #                       incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
    1361  #                       val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
    1362  #                       if val is not None:
    1363  #                               incoming['firstnames'] += u' '
    1364  #                               incoming['firstnames'] += val
    1365  #                       val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
    1366  #                       if val is not None:
    1367  #                               tmp = time.strptime(val, '%Y%m%d')
    1368  #                               incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
    1369  #                       val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)
    1370  #                       if val is not None:
    1371  #                               incoming['gender'] = val
    1372  #                       incoming['external_data_id'] = filename
    1373
    1374          #-------------------------------------------------------
    1375          #test_import_HL7(sys.argv[2])
    1376          #test_xml_extract()
    1377          #test_stage_hl7_from_xml()
    1378          #test_stage_hl7()
    1379          #test_format_hl7_message()
    1380          #test_format_hl7_file(sys.argv[2])
    1381          #test___fix_malformed_hl7()
    1382          #test_split_hl7_file()
    1383          test_parse_hl7()
    1384