/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
 * JFlex 1.4.2                                                             *
 * Copyright (C) 1998-2015  Gerwin Klein <lsf@jflex.de>                    *
 * All rights reserved.                                                    *
 *                                                                         *
 * License: BSD                                                            *
 *                                                                         *
 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

package jflex;

import java.util.*;
import jflex.unicode.UnicodeProperties;

/* customizing code */

action code {:

  LexScan     scanner;
  CharClasses charClasses;
  RegExps     regExps     = new RegExps();
  Macros      macros      = new Macros();
  Integer     stateNumber;
  Timer       t           = new Timer();
  EOFActions  eofActions  = new EOFActions();
  Map<Integer,IntCharSet> preclassCache = new HashMap<Integer,IntCharSet>();

  void fatalError(ErrorMessages message, int line, int col) {
    syntaxError(message, line, col);
    throw new GeneratorException();
  }

  void fatalError(ErrorMessages message) {
    fatalError(message, scanner.currentLine(), -1);
    throw new GeneratorException();
  }

  void syntaxError(ErrorMessages message) {
    Out.error(scanner.file, message, scanner.currentLine(), -1);
  }
  
  void syntaxError(ErrorMessages message, int line) {
    Out.error(scanner.file, message, line, -1);
  }

  void syntaxError(ErrorMessages message, int line, int col) {
    Out.error(scanner.file, message, line, col);
  }

  void warning(ErrorMessages message, int line, int col) {
    Out.warning(scanner.file, message, line, col);
  }

  private boolean isDotOrNewlinePattern(RegExp r1, RegExp r2) {
    return ( r1 instanceof RegExp1 && ((RegExp1)r1).isPoint && isNewline(r2) )
        || ( r2 instanceof RegExp1 && ((RegExp1)r2).isPoint && isNewline(r1) );
  }

  private boolean isNewline(RegExp r) {
    if ( ! (r instanceof RegExp1) || null == ((RegExp1)r).content ) {
      return false;
    }
    RegExp1 r1 = (RegExp1)r;
    switch ( r1.type ) {
      case sym.STRING:
      case sym.STRING_I:
        return "\n".equals(r1.content);
      case sym.CHAR:
      case sym.CHAR_I:
        return '\n' == (char)((Integer)r1.content).intValue();
      case sym.CCLASS:
        List<Interval> intervals = (List<Interval>)r1.content;
        if ( 1 == intervals.size() ) {
          Interval i = intervals.get(0);
          return '\n' == i.start && '\n' == i.end;
        }
        return false;
      default:
        return false;
    }
  }

  private boolean check(int type, int c) {
    switch (type) {
      case sym.JLETTERCLASS:
        return Character.isJavaIdentifierStart(c);
        
      case sym.JLETTERDIGITCLASS:
        return Character.isJavaIdentifierPart(c);

      default: return false;
    }
  }
  
  private List<Interval> getPreClass(int type) {

    List<Interval> result;
    IntCharSet intCharSet = preclassCache.get(type);
    if (null != intCharSet) {
      result = intCharSet.copy().getIntervals();
    }
    else {
      UnicodeProperties unicodeProperties = scanner.getUnicodeProperties();
      switch (type) {
        case sym.LETTERCLASS: {
          intCharSet = unicodeProperties.getIntCharSet("L");
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.DIGITCLASS: {
          intCharSet = unicodeProperties.getIntCharSet("Nd");
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.DIGITCLASSNOT: {
          IntCharSet digits = unicodeProperties.getIntCharSet("Nd");
          intCharSet = new IntCharSet(new Interval(0, unicodeProperties.getMaximumCodePoint()));
          intCharSet.sub(digits);
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.UPPERCLASS: {
          // "Uppercase" is more than Uppercase_Letter, but older Unicode
          // versions don't have this definition - check for "Uppercase",
          // then fall back to Uppercase_Letter (Lu) if it does not exist.
          intCharSet = unicodeProperties.getIntCharSet("Uppercase");
          if (null == intCharSet) {
            intCharSet = unicodeProperties.getIntCharSet("Lu");
          }
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.LOWERCLASS: {
          // "Lowercase" is more than Lowercase_Letter, but older Unicode
          // versions don't have this definition - check for "Lowercase",
          // then fall back to Lowercase_Letter (Ll) if it does not exist.
          intCharSet = unicodeProperties.getIntCharSet("Lowercase");
          if (null == intCharSet) {
            intCharSet = unicodeProperties.getIntCharSet("Ll");
          }
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.WHITESPACECLASS: {
          // Although later versions do, Unicode 1.1 does not have the
          // "Whitespace" definition - check for "Whitespace", then fall back
          // to "Space_separator" (Zs) if it does not exist.
          intCharSet = unicodeProperties.getIntCharSet("Whitespace");
          if (null == intCharSet) {
            intCharSet = unicodeProperties.getIntCharSet("Zs");
          }
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.WHITESPACECLASSNOT: {
          // Although later versions do, Unicode 1.1 does not have the
          // "Whitespace" definition - check for "Whitespace", then fall back
          // to "Space_separator" (Zs) if it does not exist.
          IntCharSet whitespaceClass = unicodeProperties.getIntCharSet("Whitespace");
          if (null == whitespaceClass) {
            whitespaceClass = unicodeProperties.getIntCharSet("Zs");
          }
          intCharSet = new IntCharSet(new Interval(0, unicodeProperties.getMaximumCodePoint()));
          intCharSet.sub(whitespaceClass);
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.WORDCLASS: {
          // UTR#18: \w = [\p{alpha}\p{gc=Mark}\p{digit}\p{gc=Connector_Punctuation}]
          IntCharSet alphaClass = unicodeProperties.getIntCharSet("Alphabetic");
          if (null == alphaClass) {
            // For Unicode 1.1, substitute "Letter" (L) for "Alphabetic".
            alphaClass = unicodeProperties.getIntCharSet("L");
          }
          IntCharSet markClass = unicodeProperties.getIntCharSet("M");
          IntCharSet digitClass = unicodeProperties.getIntCharSet("Nd");
          IntCharSet connectorPunctClass = unicodeProperties.getIntCharSet("Pc");
          if (null == connectorPunctClass) {
            // For Unicode 1.1, substitute "_" for "Connector_Punctuation".
            connectorPunctClass = new IntCharSet(new Interval('_', '_'));
          }
          intCharSet = alphaClass.copy();
          intCharSet.add(markClass);
          intCharSet.add(digitClass);
          intCharSet.add(connectorPunctClass);
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.WORDCLASSNOT: {
          // UTR#18: \W = [^\p{alpha}\p{gc=Mark}\p{digit}\p{gc=Connector_Punctuation}]
          IntCharSet alphaClass = unicodeProperties.getIntCharSet("Alphabetic");
          if (null == alphaClass) {
            // For Unicode 1.1, substitute "Letter" (L) for "Alphabetic".
            alphaClass = unicodeProperties.getIntCharSet("L");
          }
          IntCharSet markClass = unicodeProperties.getIntCharSet("M");
          IntCharSet digitClass = unicodeProperties.getIntCharSet("Nd");
          IntCharSet connectorPunctClass = unicodeProperties.getIntCharSet("Pc");
          if (null == connectorPunctClass) {
            // For Unicode 1.1, substitute "_" for "Connector_Punctuation".
            connectorPunctClass = new IntCharSet(new Interval('_', '_'));
          }
          IntCharSet wordClass = alphaClass.copy();
          wordClass.add(markClass);
          wordClass.add(digitClass);
          wordClass.add(connectorPunctClass);
          intCharSet = new IntCharSet(new Interval(0, unicodeProperties.getMaximumCodePoint()));
          intCharSet.sub(wordClass);
          preclassCache.put(type, intCharSet);
          result = intCharSet.copy().getIntervals();
          break;
        }
        case sym.JLETTERCLASS:
        case sym.JLETTERDIGITCLASS:
        default: {
          result = new ArrayList<Interval>();

          int c = 0;
          int start = 0;
          int last = charClasses.getMaxCharCode();

          boolean prev, current;

          prev = check(type, 0);

          for (c = 1; c < last; c++) {

            current = check(type, c);

            if (!prev && current) start = c;
            if (prev && !current) {
              result.add(new Interval(start, c - 1));
            }

            prev = current;
          }

          // the last iteration is moved out of the loop to
          // avoid an endless loop if last == maxCharCode and
          // last+1 == 0
          current = check(type, c);

          if (!prev && current) result.add(new Interval(c, c));
          if (prev && current)  result.add(new Interval(start, c));
          if (prev && !current) result.add(new Interval(start, c - 1));

          preclassCache.put(type, new IntCharSet(result));
          break;
        }
      }
    }
    return result;
  }
  
  private RegExp makeRepeat(RegExp r, int n1, int n2, int line, int col) {

    if (n1 <= 0 && n2 <= 0) {
      syntaxError(ErrorMessages.REPEAT_ZERO, line, col);
      return null;
    }

    if (n1 > n2) {
      syntaxError(ErrorMessages.REPEAT_GREATER, line, col);
      return null;
    }
    
    int i;
    RegExp result;    

    if (n1 > 0) {
      result = r;
      n1--; n2--; // we need one concatenation less than the number of expressions to match
    }
    else {
      result = new RegExp1(sym.QUESTION,r);
      n2--;
    }

    for (i = 0; i < n1; i++) 
      result = new RegExp2(sym.CONCAT, result, r);
      
    n2-= n1;  
    for (i = 0; i < n2; i++)
      result = new RegExp2(sym.CONCAT, result, new RegExp1(sym.QUESTION,r));
    
    return result;
  }

  private RegExp makeNL() {
    List<Interval> list = new ArrayList<Interval>();
    list.add(new Interval('\n','\r'));
    list.add(new Interval('\u0085','\u0085'));
    list.add(new Interval('\u2028','\u2029'));

	// assumption: line feeds are caseless
    charClasses.makeClass(list, false);
    charClasses.makeClass('\n', false);
    charClasses.makeClass('\r', false);

    RegExp1   c = new RegExp1(sym.CCLASS, list);
    Integer n = new Integer((int)'\n');
    Integer r = new Integer((int)'\r');

    return new RegExp2(sym.BAR, 
                       c, 
                       new RegExp2(sym.CONCAT, 
                                   new RegExp1(sym.CHAR, r), 
                                   new RegExp1(sym.CHAR, n)));
  }
  
:};

parser code {:
   public LexScan scanner;

   public LexParse(LexScan scanner) {
     super(scanner);
     this.scanner = scanner;
   }
   
   public CharClasses getCharClasses() {
     return action_obj.charClasses;
   }

   public EOFActions getEOFActions() {
     return action_obj.eofActions;
   }
       
   public void report_error(String message, Object info) {     
     if ( info instanceof java_cup.runtime.Symbol ) {
       java_cup.runtime.Symbol s = (java_cup.runtime.Symbol) info;
  
       if (s.sym == sym.EOF) 
         Out.error(ErrorMessages.UNEXPECTED_EOF);
       else
         Out.error(scanner.file, ErrorMessages.SYNTAX_ERROR, s.left, s.right);
     }
     else 
       Out.error(ErrorMessages.UNKNOWN_SYNTAX);
   }
   
   public void report_fatal_error(String message, Object info) {
     // report_error(message, info);
     throw new GeneratorException();
   }
:};

init with {:
  action_obj.scanner = scanner;
  // CharClasses.init() is delayed until UnicodeProperties.init() has been called,
  // since the max char code won't be known until then.
  action_obj.charClasses = scanner.getCharClasses();
:};

/* token declarations */

terminal OPENBRACKET, CLOSEBRACKET, HAT, DOLLAR, OPENCLASS,
         CLOSECLASS, DASH, DELIMITER, EQUALS, COMMA, LESSTHAN,
         MORETHAN, LBRACE, RBRACE, ASCII, FULL, UNICODE, REGEXPEND;

terminal JLETTERCLASS, JLETTERDIGITCLASS, LETTERCLASS, DIGITCLASS,
         DIGITCLASSNOT, UPPERCLASS, LOWERCLASS, EOFRULE, NOACTION, LOOKAHEAD,
         WHITESPACECLASS, WHITESPACECLASSNOT, WORDCLASS, WORDCLASSNOT,
         UNION, INTERSECTION, DIFFERENCE, SYMMETRICDIFFERENCE;
         
terminal Action ACTION;
terminal String IDENT, USERCODE;
terminal Integer REPEAT;

/* tokens used in RegExp parse tree */
terminal STAR, PLUS, BAR, QUESTION, POINT, NEWLINE, BANG, TILDE;

terminal Integer CHAR;
terminal String STRING, MACROUSE;
terminal IntCharSet UNIPROPCCLASS, UNIPROPCCLASSNOT;

/* symbols *only* used in the parse tree (not in the grammar) */
terminal CCLASS, CCLASSNOT, CONCAT;
terminal STRING_I, CHAR_I;  /* case insensitive strings/chars */


non terminal           macros, macro;
non terminal Integer   rule;
non terminal NFA       specification;
non terminal RegExp    series, concs, nregexp, regexp, charclass, lookahead;
non terminal Interval  classcontentelem;
non terminal List<Interval> classcontent, preclass;
non terminal List<Integer> states, statesOPT, rules;
non terminal Boolean   hatOPT;
non terminal Action    act, actions;


/* grammar specification */
start with specification;

specification ::=  USERCODE
                   /* delimiter is checked in lexer */
                   macros
                   DELIMITER 
                   rules
                   {:
                     scanner.t.stop();

                     Out.checkErrors();
                     
                     Out.time(ErrorMessages.PARSING_TOOK, t);
                     
                     macros.expand();
                     if (Options.unused_warning) {
	                     for (String unusedMacro : macros.unused()) {
	                       Out.warning(String.format(
	                    		   "Macro \"%s\" has been declared but never used.", unusedMacro));
	                     }
                     }

                     SemCheck.check(regExps, macros, scanner.file);
  
                     regExps.checkActions();
                     regExps.checkLookAheads();

                     Out.checkErrors();

                     if (Options.dump) charClasses.dump();

                     Out.print("Constructing NFA : ");

                     t.start();
                     int num = regExps.getNum();
                     
                     RESULT = new NFA(charClasses.getNumClasses(), 
                                      scanner, regExps, macros, charClasses);
                     
                     eofActions.setNumLexStates(scanner.states.number());

                     for (int i = 0; i < num; i++) {
                       if (regExps.isEOF(i))
                         eofActions.add( regExps.getStates(i), regExps.getAction(i) );
                       else
                         RESULT.addRegExp(i);
                     }
                     
                     if (scanner.standalone) RESULT.addStandaloneRule();
                     t.stop();
       
                     Out.time("");              
	                 Out.time(ErrorMessages.NFA_TOOK, t);
                     
                   :}
                 | /* emtpy spec. error */
                   {: 
                     fatalError(ErrorMessages.NO_LEX_SPEC);
                   :}
                 ;  

macros        ::=  /* empty, most switches & state declarations are parsed in lexer */
                | macros macro
                | error;

macro         ::=  ASCII
                   {: charClasses.setMaxCharCode(127); :}
                |  FULL
                   {: charClasses.setMaxCharCode(255); :}
                |  UNICODE
                   {: /* max char code is set via CharClasses.init() */ :} 
                |  IDENT:name EQUALS series:definition REGEXPEND
                   {: macros.insert(name, definition); :} 
                | IDENT EQUALS:e
                   {: syntaxError(ErrorMessages.REGEXP_EXPECTED, eleft, eright); :}
                ;
                

rules         ::=  rules:rlist rule:r
                   {: rlist.add(r); RESULT = rlist; :}
                |  rules:rlist1 LESSTHAN states:states MORETHAN LBRACE rules:rlist2 RBRACE
                   {:
                     for (Integer elem : rlist2) {
                       // might be null for error case of "rule"
                       if (elem != null) { 
                         regExps.addStates( elem, states );
                       }
                       rlist1.add( elem );
                     }                       
                     RESULT = rlist1;
                   :}
                |  LESSTHAN states:states MORETHAN LBRACE rules:rlist RBRACE
                   {: 
                     for (Integer elem : rlist) {
                       // might be null for error case of "rule"
                       if (elem != null) { 
                         regExps.addStates( elem, states );
                       }
                     }
                     RESULT = rlist;
                   :}
                |  rule:r
                   {: RESULT = new ArrayList<Integer>(); RESULT.add(r); :}
                ;  

rule          ::=  statesOPT:s hatOPT:bol series:r actions:a
                   {: RESULT = regExps.insert(rleft, s, r, a, bol, null); :}
                |  statesOPT:s hatOPT:bol series:r lookahead:l act:a
                   {: RESULT = regExps.insert(rleft, s, r, a, bol, l); :}
                |  statesOPT:s hatOPT:bol series:r lookahead:l NOACTION:a
                   {: syntaxError(ErrorMessages.LOOKAHEAD_NEEDS_ACTION, aleft, aright+1); :}
                |  statesOPT:s EOFRULE ACTION:a
                   {: RESULT = regExps.insert(s, a); :}
                |  error
                ;

lookahead     ::=  DOLLAR
                   {: RESULT = makeNL(); :}
                |  LOOKAHEAD series:r
                   {: RESULT = r; :}
                |  LOOKAHEAD series:s DOLLAR
                   {: RESULT = new RegExp2(sym.CONCAT, s, makeNL()); :}
                ;

act           ::=  REGEXPEND ACTION:a
                   {: RESULT = a; :}
				;

actions       ::=  act:a
                   {: RESULT = a; :}
                |  NOACTION
                ;

                
statesOPT     ::=  LESSTHAN states:list MORETHAN          
                   {: RESULT = list; :}
                |  /* empty */
                   {: RESULT = new ArrayList<Integer>(); :}                   
                ;
                
states        ::=  IDENT:id COMMA states:list
                   {:
                     stateNumber = scanner.states.getNumber( id );
                     if ( stateNumber != null )
                       list.add( stateNumber ); 
                     else {
                       throw new ScannerException(scanner.file, ErrorMessages.LEXSTATE_UNDECL, 
                                                  idleft, idright);
                     }
                     RESULT = list;
                   :}                
                |  IDENT:id
                   {:
                     List<Integer> list = new ArrayList<Integer>();
                     stateNumber = scanner.states.getNumber( id );
                     if ( stateNumber != null )
                       list.add( stateNumber ); 
                     else {
                       throw new ScannerException(scanner.file, ErrorMessages.LEXSTATE_UNDECL, 
                                                  idleft, idright);
                     }
                     RESULT = list;
                   :}
                 | IDENT COMMA:c
                   {: syntaxError(ErrorMessages.REGEXP_EXPECTED, cleft, cright+1); :}
                ;
                
hatOPT        ::=  HAT 
                   {:
                      // assumption: newline chars have no uppercase variant
                      charClasses.makeClass("\n\r\u000B\u000C\u0085\u2028\u2029", false);
                      RESULT = true;
                   :}
                |  /* empty */ 
                   {: RESULT = false; :}
                ;

series        ::= series:r1 BAR concs:r2
                  {:
                     if ( ! Options.jlex && ! Options.legacy_dot && isDotOrNewlinePattern(r1, r2) ) {
                       warning(ErrorMessages.DOT_BAR_NEWLINE_DOES_NOT_MATCH_ALL_CHARS, r1left, r1right);
                     }
                     RESULT = new RegExp2(sym.BAR, r1, r2);
                  :}
                | concs:r
                  {: RESULT = r; :} 
                | BAR:b
                  {: syntaxError(ErrorMessages.REGEXP_EXPECTED, bleft, bright); :}
                ;
               
concs         ::= concs:r1 nregexp:r2
                  {: RESULT = new RegExp2(sym.CONCAT, r1, r2); :} 
                | nregexp:r
                  {: RESULT = r; :}
                ;

nregexp       ::= regexp:r
                  {: RESULT = r; :}
                | BANG nregexp:r
                  {: RESULT = new RegExp1(sym.BANG, r); :}
                | TILDE nregexp:r
                  {: RESULT = new RegExp1(sym.TILDE, r); :}
                ;

regexp        ::=  regexp:r STAR
                   {: RESULT = new RegExp1(sym.STAR, r); :}
                |  regexp:r PLUS
                   {: RESULT = new RegExp1(sym.PLUS, r); :}
                |  regexp:r QUESTION
                   {: RESULT = new RegExp1(sym.QUESTION, r); :}
                |  regexp:r REPEAT:n RBRACE:b
                   {: RESULT = makeRepeat(r, n, n, bleft, bright); :}
                |  regexp:r REPEAT:n1 REPEAT:n2 RBRACE
                   {: RESULT = makeRepeat(r, n1, n2, n1left, n2right); :}
                |  OPENBRACKET series:r CLOSEBRACKET
                   {: RESULT = r; :}
                |  MACROUSE:ident
                   {: 
                      if ( !scanner.macroDefinition ) {
                        if ( ! macros.markUsed(ident) ) 
                          throw new ScannerException(scanner.file, ErrorMessages.MACRO_UNDECL, 
                                                     identleft, identright);
                      }
                      RESULT = new RegExp1(sym.MACROUSE, ident); 
                   :}
                |  charclass:c
                   {: RESULT = c; :}
                |  preclass:list
                   {:
                     try {
                       // assumption [correct?]: preclasses are already closed under case
                       charClasses.makeClass(list, false);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, listleft);
                     }
                     RESULT = new RegExp1(sym.CCLASS, list);
                   :}
                |  UNIPROPCCLASS:intcharset
                   {:
                     IntCharSet set
                       = scanner.caseless 
                       ? intcharset.getCaseless(scanner.getUnicodeProperties()) 
                       : intcharset.copy();
                     try {
                       charClasses.makeClass(set.getIntervals(), false);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, intcharsetleft, intcharsetright);
                     }
                     RESULT = new RegExp1(sym.CCLASS, set.getIntervals());
                   :}
                |  UNIPROPCCLASSNOT:notintcharset
                   {:
                     IntCharSet set
                       = scanner.caseless 
                       ? notintcharset.getCaseless(scanner.getUnicodeProperties()) 
                       : notintcharset.copy();
                     try {
                       charClasses.makeClassNot(set.getIntervals(), false);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, notintcharsetleft, notintcharsetright);
                     }
                     RESULT = new RegExp1(sym.CCLASSNOT, set.getIntervals());
                   :}
                |  STRING:str
                   {: 
                     try {
                       if ( scanner.caseless ) {
                         charClasses.makeClass(str, true);
                         RESULT = new RegExp1(sym.STRING_I, str);
                       }
                       else {
                         charClasses.makeClass(str, false);
                         RESULT = new RegExp1(sym.STRING, str); 
                       }
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CS2SMALL_STRING, strleft, strright);
                     }

                   :}
                |  POINT
                   {: 
                      List<Interval> any = new ArrayList<Interval>();
                      if ( Options.jlex || Options.legacy_dot ) {
                        any.add(new Interval('\n','\n'));
                        // assumption: newline chars have no uppercase variant
                        charClasses.makeClass('\n', false);
                      }
                      else {
                        any.add(new Interval('\n','\r'));
                        any.add(new Interval('\u0085','\u0085'));
                        any.add(new Interval('\u2028','\u2029'));
                        // assumption: newline chars have no uppercase variant
                        charClasses.makeClass("\n\r\u000B\u000C\u0085\u2028\u2029", false);
                      }
                      RegExp1 r = new RegExp1(sym.CCLASSNOT, any);
                      r.isPoint = true;
                      RESULT = r;
                   :}
                |  NEWLINE
                   {: RESULT = makeNL(); :}
                |  CHAR:c
                   {: 
                     try {
                       if ( scanner.caseless ) {
                         charClasses.makeClass(c, true);
                         RESULT = new RegExp1(sym.CHAR_I, c);
                       }
                       else {
                         charClasses.makeClass(c, false);
                         RESULT = new RegExp1(sym.CHAR, c); 
                       }
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CS2SMALL_CHAR, cleft, cright);
                     }
                   :}
                ;

charclass     ::=  OPENCLASS CLOSECLASS
                   {: 
                     RESULT = new RegExp1(sym.CCLASS,null);
                   :}
                |  OPENCLASS classcontent:list CLOSECLASS:close
                   {:
                     try {
                       charClasses.makeClass(list, Options.jlex && scanner.caseless);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
                     }
                     RESULT = new RegExp1(sym.CCLASS,list);
                   :}
                |  OPENCLASS HAT CLOSECLASS:close
                   {: 
                     List<Interval> list = new ArrayList<Interval>();
                     list.add(new Interval(0, charClasses.getMaxCharCode()));
                     try {
                       charClasses.makeClass(list, false);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
                     }
                     RESULT = new RegExp1(sym.CCLASS,list);
                   :}
                |  OPENCLASS HAT classcontent:list CLOSECLASS:close
                   {: 
                     try {
                       charClasses.makeClassNot(list, Options.jlex && scanner.caseless);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
                     }
                     RESULT = new RegExp1(sym.CCLASSNOT,list);
                   :}
                | OPENCLASS DASH classcontent:list CLOSECLASS:close
                   {: 
                     try {
                       list.add(new Interval('-','-'));
                       charClasses.makeClass(list, Options.jlex && scanner.caseless);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
                     }
                     RESULT = new RegExp1(sym.CCLASS,list);
                   :}
                |  OPENCLASS HAT DASH classcontent:list CLOSECLASS:close
                   {: 
                     try {
                       list.add(new Interval('-','-'));
                       charClasses.makeClassNot(list, Options.jlex && scanner.caseless);
                     }
                     catch (CharClassException e) {
                       syntaxError(ErrorMessages.CHARSET_2_SMALL, closeleft, closeright);
                     }
                     RESULT = new RegExp1(sym.CCLASSNOT,list);
                   :}
                |  OPENCLASS classcontent:list1 INTERSECTION classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     List<Interval> intervals = intcharset1.and(intcharset2).getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 INTERSECTION classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     List<Interval> intervals = intcharset1.and(intcharset2).getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS classcontent:list1 INTERSECTION DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     List<Interval> intervals = intcharset1.and(intcharset2).getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 INTERSECTION DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     List<Interval> intervals = intcharset1.and(intcharset2).getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 INTERSECTION classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     List<Interval> intervals = intersection.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 INTERSECTION classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     List<Interval> intervals = intersection.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 INTERSECTION DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     List<Interval> intervals = intersection.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 INTERSECTION DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     List<Interval> intervals = intersection.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS classcontent:list1 DIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 DIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS classcontent:list1 DIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 DIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 DIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 DIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 DIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 DIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     // IntCharSet.sub() assumes its argument is a subset, so subtract intersection
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     intcharset1.sub(intersection);
                     List<Interval> intervals = intcharset1.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS classcontent:list1 SYMMETRICDIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 SYMMETRICDIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS classcontent:list1 SYMMETRICDIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS DASH classcontent:list1 SYMMETRICDIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClass(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASS, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 SYMMETRICDIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 SYMMETRICDIFFERENCE classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT classcontent:list1 SYMMETRICDIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
                |  OPENCLASS HAT DASH classcontent:list1 SYMMETRICDIFFERENCE DASH classcontent:list2 CLOSECLASS:close
                   {:
                     list1.add(new Interval('-', '-'));
                     IntCharSet intcharset1 = new IntCharSet(list1);
                     list2.add(new Interval('-', '-'));
                     IntCharSet intcharset2 = new IntCharSet(list2);
                     IntCharSet union = intcharset1.copy();
                     union.add(intcharset2);
                     IntCharSet intersection = intcharset1.and(intcharset2);
                     union.sub(intersection);
                     List<Interval> intervals = union.getIntervals();
                     charClasses.makeClassNot(intervals, Options.jlex && scanner.caseless);
                     RESULT = new RegExp1(sym.CCLASSNOT, intervals);
                   :}
               ;

classcontent  ::=  classcontent:list classcontentelem:elem
                   {:
                     list.add(elem);
                     RESULT = list;
                   :}
                |  classcontentelem:elem
                   {:
                     List<Interval> list = new ArrayList<Interval>();
                     list.add(elem);
                     RESULT = list;
                   :}
                |  classcontent:list preclass:plist
                   {:
                     list.addAll(plist);
                     RESULT = list;
                   :}
                |  preclass:list 
                   {: RESULT = list; :}
                |  classcontent:list UNIPROPCCLASS:intcharset
                   {:
                     //TODO: Is IntCharSet.copy() required here?
                     list.addAll(intcharset.copy().getIntervals());
                     RESULT = list;
                   :}
                |  UNIPROPCCLASS:intcharset
                   //TODO: Is IntCharSet.copy() required here?
                   {: RESULT = intcharset.copy().getIntervals(); :}
                |  classcontent:list UNIPROPCCLASSNOT:notintcharset
                   {:
                     IntCharSet absComp = new IntCharSet(new Interval(0, scanner.getUnicodeProperties().getMaximumCodePoint()));
                     absComp.sub(notintcharset);
                     list.addAll(absComp.getIntervals());
                     RESULT = list;
                   :}
                |  UNIPROPCCLASSNOT:notintcharset
                   {:
                      IntCharSet absComp = new IntCharSet(new Interval(0, scanner.getUnicodeProperties().getMaximumCodePoint()));
                      absComp.sub(notintcharset);
                      RESULT = absComp.getIntervals();
                   :}
                |  classcontent:list STRING:s
                   {: 
                      for (int i = 0; i < s.length(); ) {
                        int ch = s.codePointAt(i);
                        i += Character.charCount(ch);
                        list.add(new Interval(ch, ch));
                      }
                      RESULT = list;
                   :}
                |  STRING:s
                   {: 
                      RESULT = new ArrayList<Interval>();
                      for (int i = 0; i < s.length(); ) {
                        int ch = s.codePointAt(i);
                        i += Character.charCount(ch);
                        RESULT.add(new Interval(ch, ch));
                      }
                   :}
                |  classcontent:list MACROUSE:ident
                   {: 
                     syntaxError(ErrorMessages.CHARCLASS_MACRO, identleft, identright);
                   :}
                |  MACROUSE:ident
                   {: 
                     syntaxError(ErrorMessages.CHARCLASS_MACRO, identleft, identright);
                   :}
                |  classcontent:list charclass:regex
                   {:
                     List<Interval> intervals = (List<Interval>)((RegExp1)regex).content;
                     if (null != intervals) {
                       if (sym.CCLASS == regex.type) {
                         list.addAll(intervals);
                       }
                       else { //  sym.CCLASSNOT == regex.type
                         IntCharSet absComp = new IntCharSet(new Interval(0, scanner.getUnicodeProperties().getMaximumCodePoint()));
                         absComp.sub(new IntCharSet(intervals));
                         list.addAll(absComp.getIntervals());
                       }
                     }
                     RESULT = list;
                   :}
                |  charclass:regex
                   {:
                     List<Interval> list = (List<Interval>)((RegExp1)regex).content;
                     if (null == list) {
                       RESULT = new ArrayList<Interval>();
                     }
                     else {
                       if (sym.CCLASS == regex.type) {
                         RESULT = list;
                       }
                       else {
                         IntCharSet absComp = new IntCharSet(new Interval(0, scanner.getUnicodeProperties().getMaximumCodePoint()));
                         absComp.sub(new IntCharSet(list));
                         RESULT = absComp.getIntervals();
                       }
                     }
                   :}
                ;

classcontentelem ::= CHAR:c1 DASH CHAR:c2
                     {:
                       if (c2 < c1) {
                         syntaxError(ErrorMessages.IMPOSSIBLE_CHARCLASS_RANGE, c1left, c1right);
                       }
                       RESULT = new Interval(c1, c2); 
                     :}
                   | CHAR:c
                     {: RESULT = new Interval(c, c); :}
                   ;
                   
preclass ::= JLETTERCLASS
             {: RESULT = getPreClass(sym.JLETTERCLASS); :}
           | JLETTERDIGITCLASS 
             {: RESULT = getPreClass(sym.JLETTERDIGITCLASS); :}
           | LETTERCLASS
             {: RESULT = getPreClass(sym.LETTERCLASS); :}
           | DIGITCLASS
             {: RESULT = getPreClass(sym.DIGITCLASS); :}
           | DIGITCLASSNOT
             {: RESULT = getPreClass(sym.DIGITCLASSNOT); :}
           | UPPERCLASS
             {: RESULT = getPreClass(sym.UPPERCLASS); :}
           | LOWERCLASS
             {: RESULT = getPreClass(sym.LOWERCLASS); :}
           | WHITESPACECLASS
             {: RESULT = getPreClass(sym.WHITESPACECLASS); :}
           | WHITESPACECLASSNOT
             {: RESULT = getPreClass(sym.WHITESPACECLASSNOT); :}
           | WORDCLASS
             {: RESULT = getPreClass(sym.WORDCLASS); :}
           | WORDCLASSNOT
             {: RESULT = getPreClass(sym.WORDCLASSNOT); :}
           ;
