#! /usr/bin/env python

##############################################
# Remove unnecessary index entries and merge #
# other index entries to improve index       #
# usability.                                 #
#                                            #
# Author: Scott Pakin <scott.clsl@pakin.org> #
##############################################

import argparse
import collections.abc
import copy
import multiprocessing
import re
import subprocess
import sys
import textwrap
import toml
from collections import defaultdict
from dataclasses import dataclass


class Subitem():
    '''Represent an index key and an optional rendering of that key.
    The rendering is split into a rendering of the key proper, an
    optional rendering of the corresponding glyph, and optional
    suffix material.'''

    glyph_re = re.compile(r'^(.*?)\s+\((\S+)\)([^()]*)$')

    def __init__(self, key, latex=None):
        self.key = key
        self.full_render = latex

    @property
    def full_render(self):
        '''Return the render, glyph, and suffix as a single string or None
        if no rendering is defined.'''
        if self.render is None:
            return None
        if self.glyph is None:
            return self.render
        return '%s (%s)%s' % (self.render, self.glyph, self.suffix or '')

    @full_render.setter
    def full_render(self, latex):
        'Parse LaTeX code into a render, glyph, and suffix.'
        self.render = None
        self.glyph = None
        self.suffix = None
        if latex is None:
            return
        match = self.glyph_re.match(latex)
        if match is None:
            self.render = latex
        else:
            self.render = match[1]
            if match[2] != '':
                self.glyph = match[2]
                if match[3] != '':
                    self.suffix = match[3]

    def __repr__(self):
        return repr((self.key, self.render, self.glyph, self.suffix))

    def __str__(self):
        if self.render is None:
            return self.key
        if self.glyph is None:
            return f'{self.key}={self.render}'
        return '%s=%s (%s)%s' % \
            (self.key, self.render, self.glyph, self.suffix or '')


class IndexEntry():
    'Represent a single line of an .idx file.'

    entry_re = re.compile(r'^\\indexentry\{(.*)\|(.*)\}\{(\d+)\}$')
    item_re = re.compile(r'^([^=]+)(=.*)?$')
    glyph_re = re.compile(r'^(.*?)\s+\((\S+)\)$')

    def __init__(self, ln):
        # Parse the line into an item, page-number formatting code, and the
        # page number itself.  From the formatting code, extract a grouping
        # code, which is either "(", ")", or None.
        match = self.entry_re.match(ln)
        if match is None:
            raise ValueError('failed to parse %s' % repr(ln))
        item_str = match[1]
        self.group = None
        self.format = match[2]
        if self.format[0] in ['(', ')']:
            self.group = self.format[0]
            self.format = self.format[1:]
        self.page = int(match[3])
        self.trace = False   # Used only for debugging

        # Parse the item into one or more sub-items.
        sub_gt = '*GREATER-THAN*'   # Replacement for quoted ">" (i.e., "!>")
        item_str = item_str.replace('!>', sub_gt)
        subitems = [it.replace(sub_gt, '!>') for it in item_str.split('>')]

        # Convert each sub-item into a Subitem object.
        self.subitems = []
        for si in subitems:
            match = self.item_re.match(si)
            if match is None:
                raise ValueError('failed to parse %s' % repr(ln))
            render = match[2]
            if render is not None:
                render = render[1:]   # Drop the "=".
            self.subitems.append(Subitem(match[1], render))

        # To help with debugging, save the original version of the object
        # before rewrites.
        self.original = copy.deepcopy(self)

    def __repr__(self):
        return repr(([repr(si) for si in self.subitems],
                     self.group, self.format, self.page))

    def __str__(self):
        toks = []
        toks.append('\\indexentry{')
        toks.append('>'.join([str(si) for si in self.subitems]))
        toks.append('|')
        if self.group is not None:
            toks.append(self.group)
        toks.append(self.format)
        toks.append('}{%d}' % self.page)
        return ''.join(toks)

    def prioritization_key(self):
        'Return a key for use in sorting entries in priority order.'
        # The key first prioritizes entries using a small set of glyphs
        # known to look nicer than the first occurring glyph for the same
        # item.  Then, it prioritizes all non-None glyphs.  Finally, it
        # prioritizes all non-None renderings.
        subitemN = self.subitems[-1]
        n_subs = len(self.subitems)
        if all([si.render is None for si in self.subitems]):
            # Subitem tally, normal priority glyph, no glyph, no rendering.
            return (n_subs, 0, True, True)
        if all([si.glyph is None for si in self.subitems]):
            # Subitem tally, normal priority glyph, no glyph, rendering.
            return (n_subs, 0, True, False)
        for code in [
                # Nicer-looking variants
                r'$\forall$',
                r'\ABXwidering',
                r'\worldflag{',
                r'\FDSYMrightangle',
                r'\sphericalangle',
                r'\Opn',               # logix open delimiter
                r'\MNSbrace',
                r'\overbracket',
                r'\FDSYMoverbrace',
                r'\FDSYMunderbrace',
                r'\overbracket',
                r'\underbracket',
                r'\overparenthesis',
                r'\underparenthesis',
                r'\shortunderlvecc',
                r'\shortundervecc',
                r'\shortvecc',
                r'\shortlvecc',
                r'\FDSYMnsubset',
                r'\FDSYMnsupset',
                r'\FDSYMnsqsubset',
                r'\FDSYMnsqsupset',
                r'\FDSYMnSqsubset',
                r'\FDSYMnSqsupset',
                r'\FiveStar',
                r'\officialeuro',
                r'\mfWireless',
                r'\bcfeutricolore',
                r'\squadlineh',
                r'\squadlinev',
                r'\squadcross',
                r'\squaddot',
                r'\SquareCastShadowBottomRight',
                r'\WhiteSquareG',
                r'\BlackSquareG',
                r'\FDSYMemptyset',
                r'\STIXAngstrom',
                r'\textregistered',
                r'\texttrademark',
                r'\neg',
                r'\exists',
                r'\times',
                r'\invamp',
                r'\igoblackstone',
                r'\MNSrightfree',
                r'\MNSnrightfree',
                r'$\prod$',
                r'\MVLeftBracket',
                r'\faQuestionCircle[regular]',
                r'\BlackCircleG',
                r'\FDSYMmedblackdiamond',
                r'\musSegno',
                r'\MNStbigostar',
                r'\faUserCircle[regular]',
                r'\SMBTathamenorth',
                r'\usym{26A7}',   # male with stroke and male and female
                r'\faPeace',
                r'\AREVsmileface',
                r'\textcopyleft',
                r'\textcopyright',
                r'\faSignal',
                r'\faMarker',
                r'\lucideicon{panels-top-left}',

                # Arrows
                r'\twemoji{27a1}',     # "right arrow"
                r'\leftrightarrow',
                r'\$twoheadrightarrow$',
                r'\lucideicon{move-horizontal}'

                # Squares and rhombuses with arrows
                r'$\boxRight',
                r'$\boxright',
                r'$\boxdotRight',
                r'$\boxdotright',
                r'$\DiamondRight',
                r'$\Diamondright',
                r'$\DiamonddotRight',
                r'$\Diamonddotright',

                # Chess pieces
                r'\symbishop',
                r'\symking',
                r'\symknight',
                r'\sympawn',
                r'\symqueen',
                r'\symrook',

                # Halloween math
                r'\overrightwitchonbroom',
                r'\underrightwitchonbroom',
                r'\overrightwitchonpitchfork',
                r'\underrightwitchonpitchfork',
                r'\overrightswishingghost',
                r'\underrightswishingghost',
                r'\mathbat',

                # Harpoons
                r'\rightharpoonup',
                r'\STIXbarrightharpoonup',
                r'\STIXrightharpoonupbar',
                r'\ABXrightbarharpoon',
                r'\MNSnleftrightharpoonupdown',
                r'\MNSnleftrightharpoons',
                r'\MTOOLSxrightharpoonup',
                r'\longvarrightharp',
                r'\FDSYMnrightharpoonup',
                r'\ABXrightrightharpoons',

                # Corners
                r'\GOlftbotcorner',
                r'\GOlfttopcorner',
                r'\GOrtbotcorner',
                r'\GOrttopcorner',

                # Bells
                r'\faBell',
                r'\usym{1F514}',

                # Chevrons
                r'\lucideicon{chevron-right}',
                r'\lucideicon{chevrons-right}',
                r'\lucideicon{chevrons-left-right}',
                r'\lucideicon{chevrons-right-left}',
                r'\lucideicon{chevron-last}',
                r'\tiChevronRightOutline',
                r'\faCircleChevronRight',

                # Brands and logos
                r'\vscodeicon[filetype]{dartlang}',
                r'\vscodeicon[filetype]{vscode}',
                r'\vscodeicon[filetype]{typescript-official}',
                r'\vscodeicon[filetype]{rust}',
                r'\vscodeicon[filetype]{rust-toolchain}',
                r'\vscodeicon[filetype]{flutter}',
                r'\faTeX',

                # Basic Greek letters with no embellishments.
                r'$\alpha$', r'$\beta$', r'$\gamma$', r'$\delta$',
                r'$\epsilon$', r'$\zeta$', r'$\eta$', r'$\theta$',
                r'$\iota$', r'$\kappa$', r'$\lambda$', r'$\mu$', r'$\nu$',
                r'$\xi$', r'$\omicron$', r'$\pi$', r'$\rho$', r'$\sigma$',
                r'$\tau$', r'$\upsilon$', r'$\phi$', r'$\chi$', r'$\psi$',
                r'$\omega$', r'\textAlpha', r'\textBeta', r'\textGamma',
                r'\textDelta', r'\textEpsilon', r'\textZeta', r'\textEta',
                r'\textTheta', r'\textIota', r'\textKappa', r'\textLambda',
                r'\textMu', r'\textNu', r'\textXi', r'\textOmicron',
                r'\textPi', r'\textRho', r'\textSigma', r'\textTau',
                r'\textUpsilon', r'\textPhi', r'\textChi', r'\textPsi',
                r'\textOmega',
        ]:
            if code in subitemN.glyph:
                # Subitem tally, very high priority glyph, glyph, rendering.
                return (n_subs, -2, False, False)
        glyphN_lower = subitemN.glyph.lower()
        if 'arrow' in glyphN_lower and 'right' in glyphN_lower and \
           'left' not in glyphN_lower:
            # Subitem tally, high priority glyph, glyph, rendering.
            return (n_subs, -1, False, False)
        if 'triangle' in glyphN_lower and 'down' not in glyphN_lower and \
           'left' not in glyphN_lower:
            if 'up' in glyphN_lower or 'right' not in glyphN_lower:
                # Subitem tally, high priority glyph, glyph, rendering.
                return (n_subs, -1, False, False)
            else:
                # Subitem tally, medium priority glyph, glyph, rendering.
                return (n_subs, 0, False, False)
        if '[regular]' in glyphN_lower:
            # Subitem tally, high priority glyph, glyph, rendering.
            return (n_subs, -1, False, False)
        if '{light-' in glyphN_lower:
            # Subitem tally, high priority glyph, glyph, rendering.
            return (n_subs, -1, False, False)
        if '\\usym{' in glyphN_lower:
            # Subitem tally, low priority glyph, glyph, rendering.
            return (n_subs, +1, False, False)
        if '\\lucideicon{' in glyphN_lower:
            # Subitem tally, low priority glyph, glyph, rendering.
            return (n_subs, +1, False, False)
        # Subitem tally, normal priority glyph, glyph, rendering.
        return (n_subs, 0, False, False)


class EntryToTag(collections.abc.Mapping):
    "Map an entry's glyph to a unique tag."

    def __init__(self):
        self.glyph2tag = {}

    def __getitem__(self, e):
        'Given an entry, return an associated tag.'
        # Use an existing tag if available.  Otherwise create a new one.
        subitemN = e.subitems[-1]
        glyph = subitemN.glyph
        try:
            tag = self.glyph2tag[glyph]
        except KeyError:
            # Create a tag of the form "!dummy-<page>-<glyph_hash>".  The
            # leading "!" ensures that Makeindex will sort "TAG=SYM" ahead
            # of "ITEM (SYM)".  For example, under "man", a basic man
            # symbol should precede "getting haircut (<glyph>)."
            h = '%019x' % hash(glyph)
            tag = '!!dummy-%04d-%s' % \
                (e.page, h[-6:])  # Escape the "!" with a second "!".
            self.glyph2tag[glyph] = tag
        return tag

    def __iter__():
        return iter(self.glyph2tag)

    def __len__():
        return len(self.glyph2tag)


class EntryMatcher():
    'Represent a rule for matching an IndexEntry.'

    def __init__(self, rule):
        # Initialize object variables.
        self.compare_lowercase = rule.get('compare_lowercase', False)
        self.consider_all_entries = rule.get('consider_all_entries', False)
        self.trace = rule.get('trace', False)
        self.rule_type = None   # Overridden by a child class
        self.re_match = None
        have_regex = False

        # Initialize additional object variables used only for debugging.
        self.rule = rule
        self.triggered = False
        mrule = rule.get('matches', [])
        if isinstance(mrule, str):
            self.remaining_matches = set([mrule])
        else:
            self.remaining_matches = set(mrule)

        # Define a function for all other keys.  For example, define
        # self.not_prefix_fail as self._make_not_contain_failure_func
        # applied to rule['not_prefix'].
        self.subject_to_function = defaultdict(lambda: [])
        for adverb in [None, 'not']:
            for subject in [
                    None,
                    'render',
                    'top',
                    'format'
            ]:
                for verb in ['matches', 'contains', 'prefix', 'regex']:
                    toml_key = '_'.join([c
                                         for c in [adverb, subject, verb]
                                         if c is not None])
                    try:
                        toml_value = rule[toml_key]
                        factory_name = '_make_%s_failure_func' % \
                            '_'.join([c
                                      for c in [adverb, verb]
                                      if c is not None])
                        factory = getattr(self, factory_name)
                        func = factory(toml_value)
                        self.subject_to_function[subject].append(func)
                        if verb == 'regex':
                            if have_regex:
                                raise RuntimeError('more than one regex'
                                                   ' was used in rule %s' %
                                                   rule)
                            have_regex = True
                    except KeyError:
                        pass

        # Abort the program if there is not at least one function in the list.
        if len(self.subject_to_function) == 0:
            raise RuntimeError('no matching patterns found in rule %s' % rule)

    @staticmethod
    def _succeed(item):
        '''Return a function that itself always returns False (no match
        failure).'''
        return False

    @staticmethod
    def _make_matches_failure_func(pattern):
        'Return a function that itself returns True on a match failure.'
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: item != pattern
        else:
            pattern = set(pattern)
            return lambda item: all([item != p for p in pattern])

    @staticmethod
    def _make_not_matches_failure_func(pattern):
        'Return a function that itself returns False on a match failure.'
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: item == pattern
        else:
            pattern = set(pattern)
            return lambda item: any([item == p for p in pattern])

    @staticmethod
    def _make_contains_failure_func(pattern):
        '''Return a function that itself returns True on a containing
        failure.'''
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: pattern not in item
        else:
            return lambda item: all([p not in item for p in pattern])

    @staticmethod
    def _make_not_contains_failure_func(pattern):
        '''Return a function that itself returns False on a containing
        failure.'''
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: pattern in item
        else:
            return lambda item: any([p in item for p in pattern])

    @staticmethod
    def _make_prefix_failure_func(pattern):
        'Return a function that itself returns True on a prefix failure.'
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: not item.startswith(pattern)
        else:
            return lambda item: all([not item.startswith(p) for p in pattern])

    @staticmethod
    def _make_not_prefix_failure_func(pattern):
        'Return a function that itself returns False on a prefix failure.'
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            return lambda item: item.startswith(pattern)
        else:
            return lambda item: any([item.startswith(p) for p in pattern])

    def _make_regex_failure_func(self, pattern):
        '''Return a function that itself returns True on a regular-expression
        match failure.'''
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            regex = re.compile(pattern)
            return lambda item: (setattr(self, 're_match', regex.search(item)),
                                 self.re_match is None)[1]
        else:
            raise ValueError('lists of regular expressions are not supported')

    def _make_not_regex_failure_func(self, pattern):
        '''Return a function that itself returns False on a regular-expression
        match failure.'''
        if pattern is None:
            return lambda item: False
        elif isinstance(pattern, str):
            regex = re.compile(pattern)
            return lambda item: (setattr(self, 're_match', regex.search(item)),
                                 self.re_match is not None)[1]
        else:
            raise ValueError('lists of regular expressions are not supported')

    def matches_pattern(self, entry):
        'Return True if the entry matches the pattern.'
        # Extract the first item, final item, and final rendering code.
        top_item = entry.subitems[0].key
        item = entry.subitems[-1].key
        try:
            full_render = entry.subitems[-1].full_render or ''
        except IndexError:
            full_render = ''
        if self.compare_lowercase:
            top_item = top_item.lower()
            item = item.lower()
            full_render = full_render.lower()
        format = entry.format or ''

        # Ignore non-symbol entries unless consider_all_entries is True.
        if not self.consider_all_entries:
            if format.startswith('hyperindexformat') or \
               format.startswith('hyperxindexformat'):
                # "see" and "see also"
                return False
            if '\\href' in full_render:
                # Package or other hyperlink
                return False

        # Return False if we do not match the IndexEntry.
        for subject, arg in [
                (None, item),
                ('render', full_render),
                ('top', top_item),
                ('format', format)
        ]:
            for match_func in self.subject_to_function[subject]:
                if match_func(arg):
                    return False

        # The rule matched the IndexEntry.  Provide trace output if
        # requested for either the entry or the rule.
        if entry.trace or self.trace:
            sys.stderr.write('=== Trace ===\n')
            if entry.trace:
                sys.stderr.write('Trigger: entry\n')
            else:
                sys.stderr.write('Trigger: rule\n')
            sys.stderr.write('Entry:\n    %s\n' % str(entry))
            sys.stderr.write('Rule:\n')
            rule_str = '[[%s]]\n' % self.rule_type
            rule_str += toml.dumps(self.rule)
            sys.stderr.write(textwrap.indent(rule_str, '    '))

        # As a special case for simple matches, discard the alternative
        # that matched.
        self.remaining_matches.discard(item)

        # Return True, and record that the rule was triggered.
        self.triggered = True
        return True


class EntryTracer(EntryMatcher):
    'Represent a single rule for tracing an IndexEntry.'

    def __init__(self, rule):
        super().__init__(rule)
        self.rule_type = 'trace'


class EntryDeleter(EntryMatcher):
    'Represent a single rule for deleting an IndexEntry.'

    def __init__(self, rule):
        super().__init__(rule)
        self.rule_type = 'delete'


class EntryRewriter(EntryMatcher):
    'Represent a single rule for rewriting an IndexEntry.'

    # Split a string into up to 10 fields.  The 11th group contains all
    # remaining fields.
    fields_re = re.compile(r'(\S+)' + r'((?:\s+)\S+)?'*9 + r'(.*)$')

    def __init__(self, rule):
        super().__init__(rule)
        self.rule_type = 'rewrite'
        self.item = rule.get('item', None)
        self.word = rule.get('word', None)
        self.render = rule.get('render', None)
        self.lowercase_item = rule.get('lowercase_item', False)
        self.lowercase_word = rule.get('lowercase_word', False)
        self.uppercase_word = rule.get('uppercase_word', False)
        self.capitalize_word = rule.get('capitalize_word', False)
        self.pluralize_word = rule.get('pluralize_word', False)
        self.preserve_escapes = rule.get('preserve_escapes', False)
        if sum([int(e) for e in [self.lowercase_word,
                                 self.uppercase_word,
                                 self.capitalize_word]]) > 1:
            raise ValueError('lowercase_word, uppercase_word, and'
                             ' capitalize_word are mutually exclusive')
        self.format = rule.get('format', None)
        self.see = rule.get('see', None)
        self.seealso = rule.get('seealso', None)
        self.stop_on_match = not rule.get('continue', False)
        self.convert_numbers = rule.get('convert_numbers', False)
        self.number_words = self.construct_number_words()
        if sum([int(e is not None) for e in [self.format,
                                             self.see,
                                             self.seealso]]) > 1:
            raise ValueError('format, see, and seealso are mutually exclusive')

    @staticmethod
    def construct_number_words(number_words=[]):
        '''Construct names for numbers as a reverse-sorted list of
        {number, name} tuples.'''
        # If we were called previously, return the list from before.
        if number_words != []:
            return number_words

        # Define base names.
        units = [
            "zero", "one", "two", "three", "four", "five", "six", "seven",
            "eight", "nine"
        ]
        specials = [
            "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen",
            "sixteen", "seventeen", "eighteen", "nineteen"
        ]
        tens = [
                "twenty", "thirty", "forty", "fifty", "sixty",
                "seventy", "eighty", "ninety"
        ]
        roman_units = [
            "i", "ii", "iii", "iv", "v", "vi", "vii", "viii", "ix"
        ]
        roman_tens = [
            "x", "xx", "xxx", "xl", "l", "lx", "lxx", "lxxx", "xc"
        ]
        roman_hundreds = [
            "c", "cc", "ccc", "cd", "d", "dc", "dcc", "dccc", "cm", "m"
        ]

        # Construct names for various Roman numerals.
        number_words.extend(zip([str(d) for d in range(1, 10)], roman_units))
        number_words.extend(zip([str(d) for d in range(11, 20)],
                                ['x' + r for r in roman_units]))
        number_words.extend(zip([str(d*10) for d in range(1, 10)], roman_tens))
        number_words.extend(zip([str(d*100) for d in range(1, 11)],
                                roman_hundreds))

        # Construct names for the numbers 0-99.
        number_words.extend(zip([str(d) for d in range(20)], units + specials))
        num = 19
        for t in tens:
            num += 1
            number_words.append((str(num), t))
            for u in units[1:]:
                num += 1
                number_words.append((str(num), t + u))

        # Construct names for multiples of 100 from 100 to 900.
        for i, u in enumerate(units):
            if i == 0:
                continue
            number_words.append((str(i*100), u + 'hundred'))

        # Construct names for multiples of 1000 from 1000 to 9000.
        for i, u in enumerate(units):
            if i == 0:
                continue
            number_words.append((str(i*1000), u + 'thousand'))

        # Construct names for multiples of 10000 from 10000 to 90000.
        for i, t in enumerate(['', 'ten'] + tens):
            if i == 0:
                continue
            number_words.append((str(i*10000), t + 'thousand'))

        # Reverse the list so longer matches occur first.
        number_words.reverse()
        return number_words

    def _maybe_lowercase_subitem(self, si):
        'Conditionally lowercase a subitem.'
        if self.lowercase_item:
            return si.lower()
        return si

    def _expand_back_refs(self, entry, s):
        '''Expand "\1", "\2", etc. within a string based on the current
        regular-expression match.'''
        'Provide different means of expanding back references.'
        # If preserve_escapes is True, return the string unmodified.
        if self.preserve_escapes:
            return s

        # If the TOML regex key was not used and no back references are
        # present, return the string unmodified.
        re_match = self.re_match
        if re_match is None and '\\' not in s:
            return s

        # If the TOML regex key was not used and back references may be
        # present, construct a regular expression that maps back
        # references to space-separated item fields, numbered starting from
        # \1 and with \g<0> representing the entire item string.
        if re_match is None:
            item = entry.subitems[-1].key
            re_match = self.fields_re.match(item)

        # Expand back references in the given string.
        try:
            return re_match.expand(s)
        except re.error:
            raise RuntimeError('rule %s failed on entry %s' %
                               (self.rule, entry))

    @staticmethod
    def _clean_whitespace(s):
        'Remove leading, trailing, and extra internal spaces.'
        return ' '.join(s.split())

    def _words_to_numbers(self, s):
        'Replace number words with numbers.'
        # Split the string based on \1.  Ignore implicit matches; require
        # explicit regex matches.
        if self.re_match[1] == self.re_match[0]:
            return s
        if s in ["Epi-Olmec", "Linear B"]:
            return s  # Ignore words containing "i" (Roman numeral 1).
        before, target, after = s.rpartition(self.re_match[1])

        # Replace the target string with a number if it matches a
        # spelled-out number or Roman numerals.
        target = target.lower()
        for num, word in self.number_words:
            if word == target:
                target = num
                break
        return before + target + after

    def rewrite(self, entry):
        '''Rewrite a single IndexEntry in place.  Return True if the entry
        matched the pattern.'''
        # Return False if we can't rewrite the IndexEntry.
        if not self.matches_pattern(entry):
            return False

        # Point to the final subitem.
        subitemN = entry.subitems[-1]

        # Rewrite the IndexEntry.  In this context, "word" corresponds to
        # Subitem.render -- the rendering of the item's key, excluding the
        # glyph and suffix.
        if self.format is not None:
            entry.format = self._expand_back_refs(entry, self.format)
        elif self.see is not None:
            entry.format = 'hyperindexformat{\\see{%s}}' % \
                self._expand_back_refs(entry, self.see)
        elif self.seealso is not None:
            entry.format = 'hyperindexformat{\\seealso{%s}}' % \
                self._expand_back_refs(entry, self.seealso)
        word = self.word

        def maybe_words_to_numbers(s):
            return self._words_to_numbers(s) if self.convert_numbers else s

        if self.item is not None:
            if isinstance(self.item, str):
                # Replace the final subitem's key.
                item = self._clean_whitespace(self.item)
                subitemN.key = \
                    maybe_words_to_numbers(
                        self._maybe_lowercase_subitem(
                            self._expand_back_refs(entry, item)
                        )
                    )
            else:
                # Replace all subitems, retaining the final glyph and suffix.
                full_render = subitemN.full_render
                entry.subitems = [
                    Subitem(
                        self._clean_whitespace(
                            maybe_words_to_numbers(
                                self._maybe_lowercase_subitem(
                                    self._expand_back_refs(entry, k)
                                )
                            )
                        )
                    )
                    for k in self.item
                ]
                subitemN = entry.subitems[-1]
                subitemN.full_render = full_render
            if word is None:
                # Specified item but unspecified word: use the final
                # subitem's key as the word.
                word = subitemN.key
        if word is not None:
            word = self._clean_whitespace(self._expand_back_refs(entry, word))
            word = maybe_words_to_numbers(word)
            if self.lowercase_word:
                word = word.lower()
            if self.uppercase_word:
                word = word.upper()
            if self.capitalize_word:
                word = word.title()
            if self.pluralize_word:
                word = word + 'es' if word[-1] == 's' else word + 's'
            if self.item is None:
                # Specified word but unspecified item: use word as the key
                # for the final subitem.
                subitemN.key = word
        if self.render is not None:
            # Specified full rendering: ignore word and use that instead.
            subitemN.full_render = self.render
        elif word is not None:
            # Specified word but unspecified rendering: use word but
            # preserve glyph and suffix.
            subitemN.render = word
        return self.stop_on_match


class EntryMerger(EntryMatcher):
    'Represent a single rule for merging IndexEntry objects.'

    merge_marker = '\\textcolor{Green}{$^+$}'  # Marker for merged symbols

    def __init__(self, state, rule):
        super().__init__(rule)
        self.rule_type = 'merge'
        self.state = state    # Map from item to representative entry

    def prepare_to_merge(self, entry):
        '''Prepare to rewrite an entry to use the same glyph as other entries
        named with the same item.'''
        # Return False if we can't merge the IndexEntry.
        if not self.matches_pattern(entry):
            return False

        # Merging two or more entries requires identical subitems matched
        # by the same rule.
        key = tuple([id(self)] + [si.key for si in entry.subitems])

        # Append the current entry to the list associated with the
        # current key.
        if entry.subitems[-1].suffix is not None:
            return True
        try:
            self.state[key].append(entry)
        except KeyError:
            self.state[key] = [entry]
        return True

    @classmethod
    def perform_all_merges(self, state):
        '''Perform all merges based on information gathered by
        prepare_to_merge.  This method should be called only once.'''
        # For any key with more than two non-")" entries, mark all entries
        # with a "merged" suffix.
        for key, entries in state.items():
            # The merge must be of two or more symbols, excluding ends of
            # ranges.
            no_close_entries = [e for e in entries if e.group != ')']
            if len(no_close_entries) < 2:
                continue

            # Find the first non-")" entry to use as the exemplar.
            for e in entries:
                if e.group == ')':
                    continue
                e.subitems[-1].suffix = self.merge_marker
                full_render = e.subitems[-1].full_render
                break

            # Copy the rendering used by the exemplar to all other entries.
            for e in entries:
                e.subitems[-1].full_render = full_render


class EntryCreator(EntryRewriter):
    '''Represent a single rule for generating new entries from existing
    entries.  This class is just like EntryRewriter except it first
    duplicates the entry.'''

    def __init__(self, rule):
        super().__init__(rule)
        self.rule_type = 'create'

    def create(self, entry):
        'Given an entry, return a new entry or None.'

        # Return None if we don't match the IndexEntry.
        if not self.matches_pattern(entry):
            return None

        # Create a new IndexEntry to return.
        new_entry = copy.deepcopy(entry)
        if not self.rewrite(new_entry):
            raise RuntimeError(f'failed to rewrite entry {new_entry}')
        if '\\see' in new_entry.format and new_entry.group in ['(', ')']:
            # Page ranges should not apply to "see" and "see also" entries.
            new_entry.group = None
        return new_entry


@dataclass
class Rules():
    'Represent a collection of rules of different types.'

    tracers:   list
    deleters:  list
    rewriters: list
    creators:  list
    mergers:   list

    def __post_init__(self):
        self.merge_state = {}


###########################################################################

def process_delimiters(all_entries):
    '''Given a list of entries, return a list of rewriting rules and a list
    of deleting rules.'''
    # Map left delimiters to right delimiters and descriptions.
    delimiter_info = {
        '(': (')', 'parenthesis'),
        '[': (']', 'square brackets'),
        'alas': ('alad', 'curly brace'),
        'Alas': ('Alad', 'curly brace'),
        'angus': ('angud', 'angle bracket'),
        'Angus': ('Angud', 'angle bracket'),
        'langlebar': ('ranglebar', 'angle bracket with bar'),
        'langledot': ('rangledot', 'angle bracket with dot'),
        'langle': ('rangle', 'angle bracket'),
        'lAngle': ('rAngle', 'double angle bracket'),
        'lbag': ('rbag', 'bag'),
        'Lbag': ('Rbag', 'bag'),
        'lblkbrbrak': ('rblkbrbrak', 'tortoise shell, filled'),
        'lbrace': ('rbrace', 'curly brace'),
        'lBrace': ('rBrace', 'curly brace with bar'),
        'lbracklltick': ('rbracklrtick', 'square bracket with lower tick'),
        'lbrack': ('rbrack', 'square bracket'),
        'lBrack': ('rBrack', 'square bracket with bar'),
        'lbrackubar': ('rbrackubar', 'square bracket with underbar'),
        'lbrackultick': ('rbrackurtick', 'square bracket with upper tick'),
        'lbrbrak': ('rbrbrak', 'tortoise shell'),
        'Lbrbrak': ('Rbrbrak', 'tortoise shell with bar'),
        'lceil': ('rceil', 'ceiling'),
        'lCeil': ('rCeil', 'double ceiling'),
        'lcorners': ('rcorners', 'corners'),
        'lcurvyangle': ('rcurvyangle', 'curved angle bracket'),
        'ldbrack': ('rdbrack', 'bracket with bar'),
        'leftevaw': ('leftevaw', 'wavy line'),
        'leftwave': ('leftwave', 'wavy line'),
        'levaw': ('revaw', 'wavy line'),
        'lfilet': ('rfilet', 'wavy line'),
        'lFloor': ('rFloor', 'double floor'),
        'lfloor': ('rfloor', 'floor'),
        'lgroup': ('rgroup', 'group brace'),
        'llangle': ('rrangle', 'angle bracket with bar'),
        'llbracket': ('rrbracket', 'bracket with bar'),
        'llceil': ('rrceil', 'double ceiling'),
        'llcorner': ('lrcorner', 'lower corners'),
        'llfloor': ('rrfloor', 'double floor'),
        'llparenthesis': ('rrparenthesis', 'double parenthesis'),
        'lmoustache': ('rmoustache', 'moustache'),
        'Lparengtr': ('Rparenless',
                      'double parenthesis with greater/less than'),
        'lparenless': ('rparengtr', 'parenthesis with less/greater than'),
        'lparen': ('rparen', 'parenthesis'),
        'lParen': ('rParen', 'parenthesis with bar'),
        'Lparen': ('Rparen', 'parenthesis with bar'),
        'lsem': ('rsem', 'bracket with bar'),
        'ltriplevert': ('rtriplevert', 'triple vertical bar'),
        'lVert': ('rVert', 'double vertical bar'),
        'lvert': ('rvert', 'vertical bar'),
        'lVvert': ('rVvert', 'triple vertical bar'),
        'Lvzigzag': ('Rvzigzag', 'double zigzag'),
        'lvzigzag': ('rvzigzag', 'zigzag'),
        'lwave': ('rwave', 'wavy line'),
        'lWavy': ('rWavy', 'double wavy line'),
        'lwavy': ('rwavy', 'wavy line'),
        'niv': ('vin', 'lower corners'),
        'quadras': ('quadrad', 'square bracket with bar'),
        'Quadras': ('Quadrad', 'square bracket with bar'),
        'textlangle': ('textrangle', 'angle bracket'),
        'textlbrackdbl': ('textrbrackdbl', 'square bracket with bar'),
        'textlquill': ('textrquill', 'quill'),
        'thickvert': ('thickvert', 'vertical bar'),
        'triple<': ('triple!>', 'triple angle bracket'),
        'triple[': ('triple]', 'triple square bracket'),
        'ulcorner': ('urcorner', 'upper corners'),
        'vert': ('vert', 'vertical bar'),
        'Vert': ('Vert', 'double bar'),
        'VERT': ('VERT', 'triple vertical bar'),
        'Vvert': ('Vvert', 'triple vertical bar'),
        'vvvert': ('vvvert', 'triple vertical bar'),
    }

    # Iterate over all entries in search of delimiters.
    rewrites = []
    deletes = []
    display_re = re.compile('^(.*)(MNS|FDSYM|STIX)d(.*)$')
    for e in all_entries:
        # Extract the key and glyph.
        subitemN = e.subitems[-1]
        glyphN = subitemN.glyph
        if glyphN is None:
            continue
        lKey = subitemN.key

        # Perform the substitution.
        rule = {}
        try:
            rKey, desc = delimiter_info[lKey]
            if lKey == 'llangle':
                # Special case for MnSymbol's \llangle, which is doubled,
                # not barred.
                if 'MNSdllangle' in glyphN:
                    desc = 'double angle bracket'
                    glyphN = '\\MNStllangle'
                rule['render_contains'] = glyphN
        except KeyError:
            continue
        lGlyph = display_re.sub(r'\1\2t\3', glyphN)
        rGlyph = lGlyph.replace(lKey, rKey)
        rGlyph = rGlyph.replace(r'\nathtriple\langle', r'\nathtriple\rangle')
        rule.update({
            'matches': lKey,
            'item': ['delimiters', desc],
            'render': '%s (%s\\graybox %s)' % (desc, lGlyph, rGlyph),
            'autogenerated': True,
        })
        rewrites.append(EntryRewriter(rule))
        if rKey != lKey:
            deletes.append(EntryDeleter({
                'matches': rKey,
                'autogenerated': True,
            }))
    return rewrites, deletes


def read_logix_rewrites():
    '''Map logix symbol names to symbol replacements by parsing the logix
    documentation.  Return a list of EntryRewriters or an empty list on
    error.'''
    # Find the logix.tex file.
    proc = subprocess.run(['kpsewhich', '--format=doc', 'logix.tex'],
                          capture_output=True, encoding='utf-8')
    if proc.returncode != 0:
        return []
    logix_tex = proc.stdout.strip()
    space_slash_re = re.compile(r'\s+/\s+')

    def patch_desc(desc):
        'Modify the item or word.'
        desc = desc.lower()
        if desc.split()[0] in ['lblack', 'lwhite']:  # Non-word
            desc = desc[1:]
        desc = desc.replace('finte', 'finite')       # Typo, I assume
        desc = desc.replace('asterick', 'asterisk')  # Spelling error
        desc = desc.replace('long ', '')             # Unnecessary
        desc = desc.replace('short ', '')            # Unnecessary
        desc = desc.replace('very ', '')             # Unnecessary
        desc = desc.replace('extra ', '')            # Unnecessary
        desc = desc.replace(' (rule)', '')           # Unnecessary
        desc = space_slash_re.sub('/', desc)         # Nicer-looking
        desc = desc.replace('plus/minus', 'plus or minus')   # Nicer-looking
        desc = desc.replace('minus/plus', 'minus or plus')   # Nicer-looking
        return desc

    # Parse the logix.tex file.
    rules = []
    with open(logix_tex) as r:
        for ln in r:
            # Map a LaTeX command to a description.
            fields = [f.strip() for f in ln.split('&')]
            if len(fields) != 3:
                continue
            item = fields[1][16:]   # Ignore "{\textbackslash}".
            desc = fields[0]
            if ' em ' in desc:
                continue
            if 'End Law' in desc:
                continue      # Don't rename *all* \End symbols to "end law".
            desc = patch_desc(desc)

            # Create an EntryRewriter corresponding to the mapping from
            # command to description.
            desc_fields = desc.split()
            if desc_fields[0] == 'circled':
                # Special case for circled symbols
                rules.append(EntryRewriter({
                    'matches': item,
                    'item': ['circled symbols', ' '.join(desc_fields[1:])],
                    'continue': True,
                    'autogenerated': True
                }))
            elif desc_fields[0] == 'open':
                # Special case for open/close delimiters
                phrase = ' '.join(desc_fields[1:])
                cls_item = 'Cls' + item[3:]
                rules.append(EntryRewriter({
                    'matches': item,
                    'item': ['delimiters', phrase],
                    'render': '%s (\\%s \\graybox \\%s)' %
                    (phrase, item, cls_item),
                    'autogenerated': True
                }))
            else:
                # Common case
                rules.append(EntryRewriter({
                    'matches': item,
                    'word': desc,
                    'continue': 'true',
                    'autogenerated': True
                }))
    return rules


def auto_merge_man_woman(state, entries):
    '''Return a list of EntryMerger objects that merge "man..." and
    "woman..." emoji.'''
    # Construct a mapping from a noun to the entries that present "man" and
    # "woman" versions of that noun.
    noun2entries = defaultdict(lambda: [])
    for e in entries:
        # Consider only twemoji symbols.
        subitemN = e.subitems[-1]
        if subitemN.glyph is None:
            continue
        if 'twemoji' not in subitemN.glyph:
            continue

        # Consider only "man NOUN" and "woman NOUN" constructs.
        fields = subitemN.key.split()
        if len(fields) != 2:
            continue
        if fields[0] not in ['man', 'woman']:
            continue
        noun2entries[fields[1]].append(e)

    # Create a new EntryMerger for all entries containing exactly one "man"
    # and one "woman" glyph.
    new_merges = []
    for noun, ents in noun2entries.items():
        glyphs = set([e.subitems[-1].glyph for e in ents])
        if len(glyphs) != 2:
            continue
        if "biking" in noun or "astronaut" in noun:
            # Other entries will be merged into these.
            continue
        genders = set([e.subitems[-1].key.split()[0] for e in ents])
        if 'man' in genders and 'woman' in genders:
            new_merges.append(EntryMerger(state, {
                'autogenerated': True,
                'matches': noun,
            }))
    return new_merges


def merge_top_levels(entries):
    'Merge "ITEM (SYM)" into "ITEM" at the top level when SYM is unique.'
    # Group all entries by top-level key.
    key2entries = defaultdict(lambda: [])
    for e in entries:
        if e.subitems[0].glyph == 'package':
            # Ignore package names.
            continue
        key2entries[e.subitems[0].key].append(e)

    # For each key associated with a unique glyph, assign that glyph
    # to all top-level subitems.
    for key, es in key2entries.items():
        # Reject certain cases.
        if len(es) == 1:
            continue     # Nothing to merge
        hyperindex, hyperpage = False, False
        for e in es:
            if len(e.subitems) != 1:
                continue    # Consider only entries with no subentries.
            if e.format.startswith('hyperpage'):
                hyperpage = True
            elif e.format.startswith('hyperindexformat'):
                hyperindex = True

        # Determine if there is a unique glyph.
        glyphs_sfxs = set()   # Pairs of {glyph, suffix}
        for e in es:
            glyph = e.subitems[0].glyph
            sfx = e.subitems[0].suffix
            if glyph is not None:
                glyphs_sfxs.add((glyph, sfx))
        if len(glyphs_sfxs) != 1:
            continue    # Glyph is not unique.
        glyph, sfx = list(glyphs_sfxs)[0]
        if '\\' not in glyph:
            continue    # Glyph is not a control word.

        # Find the entry whose rendering matches the unique glyph.
        for e in es:
            if e.subitems[0].glyph == glyph and e.subitems[0].suffix == sfx:
                full_render = e.subitems[0].full_render
                break

        # Assign the same rendering to each entry.
        for e in es:
            e.subitems[0].full_render = full_render


def reformat_same_item_different_glyphs(entries):
    '''Rewrite "ITEM=ITEM (SYM1)" and "ITEM=ITEM (SYM2)" as "ITEM" with
    subitems "SYM1" and "SYM1".'''
    # Bin entries by item name.
    unmodified_entries = []     # Entries to leave alone
    normalized_entries = []     # Entries updated from "ITEM" to "ITEM=ITEM"
    key2entries = defaultdict(lambda: [])
    for e in entries:
        if len(e.subitems) != 1:
            unmodified_entries.append(e)
            continue
        subitemN = e.subitems[-1]
        if subitemN.render is None:
            subitemN.render = subitemN.key
            normalized_entries.append(e)
            continue
        if 'href' in subitemN.render:
            unmodified_entries.append(e)
            continue
        if subitemN.glyph is None:
            unmodified_entries.append(e)
            continue
        key2entries[subitemN.key].append(e)

    # Discard entries that should not be merged.
    for item, ents in key2entries.copy().items():
        # Ignore items that repeat a single glyph.
        unique_glyphs = set([e.subitems[-1].glyph
                             for e in ents
                             if e.subitems[-1].glyph is not None])
        if len(unique_glyphs) < 2:
            unmodified_entries.extend(ents)
            del key2entries[item]
            continue

        # Ignore items with different renderings, excluding the glyph.
        common_rendering = [si.render for si in ents[0].subitems]
        for e in ents:
            rendering = [si.render for si in e.subitems]
            if rendering != common_rendering:
                unmodified_entries.extend(ents)
                del key2entries[item]
                break

    # Rewrite the remaining entries as "ITEM>TAG=SYM" with a per-glyph TAG.
    ent2tag = EntryToTag()
    for item, ents in key2entries.items():
        for e in ents:
            # Delete the glyph from the final subitem then append TAG as a
            # new subitem with rendering SYM.
            subitemN = e.subitems[-1]
            glyph = subitemN.glyph
            suffix = subitemN.suffix or ''
            subitemN.glyph = None
            tag = ent2tag[e]
            e.subitems.append(Subitem(tag, glyph + suffix))

    # Return the concatenation of the modified and unmodified index entries.
    new_entries = []
    for ents in key2entries.values():
        new_entries.extend(ents)
    return new_entries + normalized_entries + unmodified_entries


def fix_overlapping_ranges(entries):
    "Ensure that page ranges don't overlap."
    clean_entries = []
    range_state = {}    # Map from an item to its most recent entry.
    for e in all_entries:
        # Discard items that are not part of a page range.
        if len(e.subitems) != 1:
            clean_entries.append(e)
            continue
        if e.group is None:
            clean_entries.append(e)
            continue

        # Prevent adjacent open and adjacent close ranges.
        range_key = '>'.join([str(si) for si in e.subitems])
        if e.group == '(':
            # Keep the oldest open range.
            try:
                prev_entry = range_state[range_key]
                if prev_entry.group == '(':
                    # Keep the previous (open) entry; discard the current
                    # (open) entry.
                    pass
                elif prev_entry.group == ')':
                    # Keep both the previous (open) and current (closed)
                    # entries.  Remember the current entry.
                    clean_entries.append(e)
                    range_state[range_key] = e
            except KeyError:
                # First time encountering this item.
                clean_entries.append(e)
                range_state[range_key] = e
        elif e.group == ')':
            # Keep the newest close range.
            try:
                prev_entry = range_state[range_key]
                if prev_entry.group == '(':
                    # Keep both the previous (open) and current (close)
                    # entry.  Remember the current entry.
                    clean_entries.append(e)
                    range_state[range_key] = e
                elif prev_entry.group == ')':
                    # Overwrite the previous (close) entry's page number
                    # with the current (close) entry's page number.
                    prev_entry.page = e.page
            except KeyError:
                # First time encountering this item, but it's erroneous
                # (close with no open) so we discard it.
                pass
    return clean_entries


def split_second_levels(entries):
    '''If "ITEM1>ITEM2 (G1)" and "ITEM1>ITEM2 (G2)" are both encountered,
    replace them with "ITEM1>ITEM2>G1" and "ITEM1>ITEM2>G2".'''
    # Group all entries by the first two keys.  Ignore entries without
    # exactly two levels.
    keys2entries = defaultdict(lambda: [])
    for e in entries:
        if len(e.subitems) != 2:
            continue
        keys2entries[(e.subitems[0].key, e.subitems[1].key)].append(e)

    # Introduce a new level if and only if more than two entries associated
    # with a pair of keys have a parenthesized glyph.
    for entries in keys2entries.values():
        # Discard entries we can't split.
        if len(entries) == 1:
            continue
        unique_glyphs = set()
        for e in entries:
            glyph = e.subitems[-1].glyph
            if glyph is not None:
                unique_glyphs.add(glyph)
        if len(unique_glyphs) < 2:
            continue

        # Add a level to each entry containing just the glyph.
        ent2tag = EntryToTag()
        for e in entries:
            if e.subitems[-1].glyph is None:
                continue
            tag = ent2tag[e]
            render = e.subitems[1].glyph
            if e.subitems[1].suffix is not None:
                render += e.subitems[1].suffix
            si2 = Subitem(tag, render)
            e.subitems[1].full_render = e.subitems[1].render
            e.subitems.append(si2)


def promote_singletons(entries):
    '''Replace certain "ITEM1>ITEM2|see{ITEM3}" with "ITEM1|see{ITEM3}"
    entries if no other entries lie at level 2.'''
    candidates = set([
        'bishop',
        'castle',
        'king',
        'knight',
        'pawn',
        'queen',
        'rook'
    ])
    collapse = defaultdict(lambda: False, {k: True for k in candidates})
    for e in entries:
        key = e.subitems[0].key
        if key in candidates and '\\see{chess' not in e.format:
            collapse[key] = False
    for e in entries:
        if collapse[e.subitems[0].key]:
            e.subitems[0].glyph = e.subitems[1].glyph
            e.subitems[0].suffix = e.subitems[1].suffix
            e.subitems = [e.subitems[0]]


def mark_all_items(entries):
    '''Globally replace "WORD=ITEM..." with
    "WORD=\\markboth{WORD}{WORD}ITEM...".'''
    for e in entries:
        key = e.subitems[0].key
        for si in e.subitems:
            if si.render is None or si.render == '':
                si.render = si.key
            si.render = '%s\\markboth{%s}{%s}' % (si.render, key, key)


def parse_rules_from_files(toml_fnames):
    'Process all TOML files into a Rules object.'

    # Read a list of actions from each TOML-format file.
    toml_data = {
        'trace': [],
        'delete': [],
        'rewrite': [],
        'create': [],
        'merge': [],
    }
    for fn in toml_fnames:
        print('Parsing', fn)
        new_data = toml.load(fn)
        for key in toml_data:
            try:
                toml_data[key].extend(new_data[key])
            except KeyError:
                pass
    rules = Rules([], [], [], [], [])

    # Convert each trace entry to an EntryMatcher.
    try:
        for t in toml_data['trace']:
            rules.tracers.append(EntryTracer(t))
    except KeyError:
        pass

    # Convert each delete entry to an EntryDeleter.
    try:
        for d in toml_data['delete']:
            rules.deleters.append(EntryDeleter(d))
    except KeyError:
        pass

    # Convert each rewrite entry to an EntryRewriter.
    try:
        for rw in toml_data['rewrite']:
            rules.rewriters.append(EntryRewriter(rw))
    except KeyError:
        pass

    # Convert each create entry to an EntryCreator.
    try:
        for cr in toml_data['create']:
            rules.creators.append(EntryCreator(cr))
    except KeyError:
        pass

    # Convert each merge entry to an EntryMerger.
    try:
        for m in toml_data['merge']:
            rules.mergers.append(EntryMerger(rules.merge_state, m))
    except KeyError:
        pass

    # Return the collection of rules.
    return rules


def report_unused(rules):
    'Report all unused rules, excluding autogenerated ones.'
    for r in rules:
        if r.rule.get('autogenerated', False):
            continue
        if r.triggered and r.remaining_matches != set():
            sys.stderr.write('=== Partially triggered ===\n')
            sys.stderr.write('Rule:\n')
            rule_str = '[[%s]]\n' % r.rule_type
            rule_str += toml.dumps(r.rule)
            sys.stderr.write(textwrap.indent(rule_str, '    '))
            sys.stderr.write('Remaining:\n')
            rem_str = ',\n'.join([repr(m)
                                  for m in sorted(r.remaining_matches)]) + '\n'
            sys.stderr.write(textwrap.indent(rem_str, '    '))
            continue
        if not r.triggered:
            sys.stderr.write('=== Untriggered ===\n')
            sys.stderr.write('Rule:\n')
            rule_str = '[[%s]]\n' % r.rule_type
            rule_str += toml.dumps(r.rule)
            sys.stderr.write(textwrap.indent(rule_str, '    '))


###########################################################################

if __name__ == '__main__':
    # Parse the command line.
    parser = argparse.ArgumentParser(
        prog='prune-idx',
        description='Delete, merge, and rewrite index entries.')
    parser.add_argument('index_file', help='.idx file to modify')
    parser.add_argument('toml_files', nargs='+',
                        help='one or more TOML files of modification rules')
    parser.add_argument('--report-unused', action='store_true',
                        help='report rules that never triggered')
    cl_args = parser.parse_args()

    # Parse the index file into a list of IndexEntry objects.
    all_entries = []
    with open(cl_args.index_file) as r:
        for ln in r:
            all_entries.append(IndexEntry(ln))

    # Read a list of actions from all of the TOML files.
    rules = parse_rules_from_files(cl_args.toml_files)

    # Prepend logix symbols to the rewriters.  Other rewriters will rewrite
    # some of these.
    rules.rewriters = read_logix_rewrites() + rules.rewriters

    # Automatically merge "man" and "woman" emoji because they tend to look
    # extremely similar.
    rules.mergers.extend(auto_merge_man_woman(rules.merge_state, all_entries))

    # Handle delimiters specially.
    all_entries.sort(key=lambda e: e.prioritization_key())
    print('Preparing to process %d index entries' % len(all_entries))
    more_rewriters, more_deleters = process_delimiters(all_entries)
    rules.rewriters.extend(more_rewriters)
    rules.deleters.extend(more_deleters)

    def apply_tracers(entry):
        '''Apply trace-creation rules until one succeeds.  In that case, set
        the entry's trace flag to True.  Otherwise return the entry
        unmodified.'''
        for t in rules.tracers:
            if t.matches_pattern(entry):
                entry.trace = True
                return entry
        return entry

    # Trace entries for debug purposes.
    print('Applying [[trace]] rules (%d)' % len(rules.tracers))
    with multiprocessing.Pool() as pool:
        all_entries = pool.map(apply_tracers, all_entries)

    def apply_deleters(entry):
        '''Apply entry-deleting rules until one succeeds.  In that case return
        None.  Otherwise return the entry unmodified.'''
        for d in rules.deleters:
            if d.matches_pattern(entry):
                return None
        return entry

    # Delete uninteresting entries.
    print('Applying [[delete]] rules (%d)' % len(rules.deleters))
    if cl_args.report_unused:
        # Sequential but allowing rule updates
        pruned_entries = [apply_deleters(e) for e in all_entries]
        report_unused(rules.deleters)
    else:
        # Parallel but with read-only rules
        with multiprocessing.Pool() as pool:
            pruned_entries = pool.map(apply_deleters, all_entries)
    all_entries = [e for e in pruned_entries if e is not None]

    # Additionally remove all "see" and "see also" entries that point
    # to a control sequence.
    all_entries = [e
                   for e in all_entries
                   if (r'\see{\texttt{\string' not in e.format and
                       r'\seealso{\texttt{\string' not in e.format)]

    def apply_rewriters(entry):
        'Apply entry-rewriting rules until one succeeds.'
        for rw in rules.rewriters:
            if rw.rewrite(entry):
                return entry
        return entry

    # Apply the rewriting actions to each entry, stopping on the first match.
    print('Applying [[rewrite]] rules (%d)' % len(rules.rewriters))
    if cl_args.report_unused:
        # Sequential but allowing rule updates
        all_entries = [apply_rewriters(e) for e in all_entries]
        report_unused(rules.rewriters)
    else:
        # Parallel but with read-only rules
        with multiprocessing.Pool() as pool:
            all_entries = pool.map(apply_rewriters, all_entries)

    # Rewrite "NUM..." to use only "NUM" as the key.  Doing so indexes
    # terms like "8 ball" under Numbers instead of Symbols.
    initial_num_re = re.compile(r'^(\d+)(\D.*)$')
    for e in all_entries:
        match = initial_num_re.match(e.subitems[0].key)
        if match is None:
            continue
        e.subitems.insert(0, Subitem(match[1]))

    def apply_creators(entry):
        'Apply all entry-creating rules to the current entry.'
        new_entries = []
        for cr in rules.creators:
            e = cr.create(entry)
            if e is not None:
                new_entries.append(e)
        return new_entries

    # Apply the creating actions to each entry, stopping on the first match.
    print('Applying [[create]] rules (%d)' % len(rules.creators))
    if cl_args.report_unused:
        # Sequential but allowing rule updates
        all_entries.extend([e
                            for es in [apply_creators(entry)
                                       for entry in all_entries]
                            for e in es])
        report_unused(rules.creators)
    else:
        # Parallel but with read-only rules
        with multiprocessing.Pool() as pool:
            all_entries.extend([e
                                for es in pool.map(apply_creators, all_entries)
                                for e in es])

    # The next few stanzas all pertain to merging.
    print('Applying [[merge]] rules (%d)' % len(rules.mergers))

    # Move index entries with no specified glyph or rendering to the end of
    # the list so merges see any glyphs first.  Also, move certain entries
    # to the beginning of the list so their glyphs are used instead of the
    # glyph that appears first in the document.
    all_entries.sort(key=lambda e: e.prioritization_key())

    # Remove duplicates before merging.  Otherwise, duplicate entries will
    # be marked as having multiple glyphs when they in fact have only one.
    entries_seen = set()
    unique_entries = []
    for e in all_entries:
        e_str = str(e)
        if e_str in entries_seen:
            continue
        unique_entries.append(e)
        entries_seen.add(e_str)
    all_entries = unique_entries

    # Merge glyphs that are essentially just font variants, stopping on the
    # first match for each entry.
    for e in all_entries:
        for m in rules.mergers:
            if m.prepare_to_merge(e):
                break
    if cl_args.report_unused:
        report_unused(rules.mergers)
    EntryMerger.perform_all_merges(rules.merge_state)

    # Replace top-level "ITEM" entries with "ITEM (SYM)" when both exist
    # and SYM is unique.
    merge_top_levels(all_entries)

    # The next few stanzas all pertain to cleanup.
    print('Cleaning up the index')

    # Use subitems for different glyphs corresponding to the same item.
    all_entries = reformat_same_item_different_glyphs(all_entries)

    # Clean up the index by simplifying "ITEM=ITEM" to just "ITEM".  In many
    # cases this enables Makeindex to merge entries with and without subitems.
    for e in all_entries:
        for si in e.subitems:
            if si.key == si.full_render:
                si.full_render = None

    # Clean up the index by making all "see" and "see also" entries point
    # to the same page (0).  The prevents Makeindex from generating
    # multiple "see" or "see also" entries per entry.
    for e in all_entries:
        if e.format.startswith("hyperindexformat{\\see"):
            e.page = 0

    # Fix overlapping page ranges.
    all_entries = fix_overlapping_ranges(all_entries)

    # Split repeated second-level terms into third levels.
    split_second_levels(all_entries)

    # Insert a \markboth{...}{...} into every entry.
    mark_all_items(all_entries)

    # Promote certain singleton second-level terms to first-level terms.
    promote_singletons(all_entries)

    # Overwrite the input file with all remaining entries.
    print('Writing %d entries' % len(all_entries))
    with open(cl_args.index_file, 'w') as w:
        for e in all_entries:
            w.write('%s\n' % str(e))
