/*
*   Copyright (c) 2000-2003, Darren Hiebert
*
*   This source code is released for free distribution under the terms of the
*   GNU General Public License version 2 or (at your option) any later version.
*
*   This module contains functions for applying regular expression matching.
*
*   The code for utilizing the Gnu regex package with regards to processing the
*   regex option and checking for regex matches was adapted from routines in
*   Gnu etags.
*/

/*
*   INCLUDE FILES
*/
#include "general.h"  /* must always come first */

#include <string.h>

#include <ctype.h>
#include <stddef.h>
#ifdef HAVE_SYS_TYPES_H
# include <sys/types.h>  /* declare off_t (not known to regex.h on FreeBSD) */
#endif
#include <regex.h>

#include <inttypes.h>

#include "debug.h"
#include "colprint_p.h"
#include "entry_p.h"
#include "field_p.h"
#include "flags_p.h"
#include "htable.h"
#include "kind.h"
#include "options.h"
#include "parse_p.h"
#include "promise.h"
#include "read.h"
#include "read_p.h"
#include "routines.h"
#include "routines_p.h"
#include "trashbox.h"
#include "xtag_p.h"

static bool regexAvailable = false;

/*
*   MACROS
*/

/* Back-references \0 through \9 */
#define BACK_REFERENCE_COUNT 10

/* The max depth of taction=enter/leave stack */
#define MTABLE_STACK_MAX_DEPTH 64

/* How many times ctags allows a mtable parser
   stays at the same input position across table switching.

   The value is derived from MTABLE_STACK_MAX_DEPTH.
   No deep meaning is in that. It just for simplifying
   Tmain cases. */
#define MTABLE_MOTIONLESS_MAX (MTABLE_STACK_MAX_DEPTH + 1)


/*
*   DATA DECLARATIONS
*/

enum pType { PTRN_TAG, PTRN_CALLBACK };

enum scopeAction {
	SCOPE_REF     = 1UL << 0,
	SCOPE_POP     = 1UL << 1,
	SCOPE_PUSH    = 1UL << 2,
	SCOPE_CLEAR   = 1UL << 3,
	SCOPE_PLACEHOLDER = 1UL << 4,
};

enum tableAction {
	TACTION_NOP,
	TACTION_ENTER,				/* {tenter=N} */
	TACTION_LEAVE,				/* {tleave} */
	TACTION_JUMP,					/* {tjump=N} */
	TACTION_RESET,				/* {treset=N} */
	TACTION_QUIT,					/* {tquit} */
};

struct fieldPattern {
	fieldType ftype;
	const char *template;
};

struct boundarySpec {
	int patternGroup;
	bool fromStartOfGroup;
	bool placeholder;
};

struct guestLangSpec {
	enum guestLangSpecType {
		GUEST_LANG_UNKNOWN,
		GUEST_LANG_PLACEHOLDER,			   /* _ */
		GUEST_LANG_STATIC_LANGNAME,		   /* C, Python,... */
		GUEST_LANG_PTN_GROUP_FOR_LANGNAME, /* \1, \2, ..., \9 */
		GUEST_LANG_PTN_GROUP_FOR_FILEMAP, /* *1, *2, ... *9 */
	} type;
	union {
		langType lang;
		int patternGroup;
	} spec;
};

struct guestSpec {
	struct guestLangSpec lang;
#define BOUNDARY_START 0
#define BOUNDARY_END  1
	struct boundarySpec boundary[2];
};

struct mGroupSpec {
#define NO_MULTILINE -1
	int forLineNumberDetermination;
	int forNextScanning;
	/* true => start, false => end */
	bool nextFromStart;
};

struct mTableActionSpec {
	enum tableAction action;
	struct regexTable *table;

	/* used when action == TACTION_ENTER */
	struct regexTable *continuation_table;
};

typedef struct {
	regex_t *pattern;
	enum pType type;
	bool exclusive;
	bool accept_empty_name;
	union {
		struct {
			int kindIndex;
			roleBitsType roleBits;
			char *name_pattern;
		} tag;
		struct {
			regexCallback function;
			void *userData;
		} callback;
	} u;
	unsigned int scopeActions;
	bool *disabled;

	enum regexParserType regptype;
	struct mGroupSpec mgroup;
	struct guestSpec guest;
	struct mTableActionSpec taction;

	int   xtagType;
	ptrArray *fieldPatterns;

	char *pattern_string;

	char *anonymous_tag_prefix;

	struct {
		errorSelection selection;
		char *message_string;
	} message;

	int refcount;
} regexPattern;


typedef struct {
	/* the pattern can be shared among entries using a refcount */
	regexPattern *pattern;

	/* but the statistics are per-table-entry */
	struct {
		unsigned int match;
		unsigned int unmatch;
	} statistics;
} regexTableEntry;


#define TABLE_INDEX_UNUSED -1
struct regexTable {
	char *name;
	ptrArray *entries;
};

struct boundaryInRequest {
	bool offset_set;
	off_t offset;
};

struct guestRequest {
	bool lang_set;
	langType lang;

	struct boundaryInRequest boundary[2];
};

struct lregexControlBlock {
	int currentScope;
	ptrArray *entries [2];

	ptrArray *tables;
	ptrArray *tstack;

	struct guestRequest *guest_req;

	langType owner;
};

/*
*   DATA DEFINITIONS
*/

/*
*   FUNCTION DEFINITIONS
*/
static int getTableIndexForName (const struct lregexControlBlock *const lcb, const char *name);
static void deletePattern (regexPattern *p);
static int  makePromiseForAreaSpecifiedWithOffsets (const char *parser,
													off_t startOffset,
													off_t endOffset);

static struct guestRequest *guestRequestNew (void);
static void   guestRequestDelete (struct guestRequest *);
static bool   guestRequestIsFilled(struct guestRequest *);
static void   guestRequestClear (struct guestRequest *);
static void   guestRequestSubmit (struct guestRequest *);

static void deleteTable (void *ptrn)
{
	struct regexTable *t = ptrn;

	ptrArrayDelete (t->entries);
	eFree (t->name);
	eFree (t);
}

static void deleteTableEntry (void *ptrn)
{
	regexTableEntry *e = ptrn;
	Assert (e && e->pattern);
	deletePattern (e->pattern);
	eFree (e);
}

static void deletePattern (regexPattern *p)
{
	p->refcount--;

	if (p->refcount > 0)
		return;

	regfree (p->pattern);
	eFree (p->pattern);
	p->pattern = NULL;

	if (p->type == PTRN_TAG)
	{
		eFree (p->u.tag.name_pattern);
		p->u.tag.name_pattern = NULL;
	}

	if (p->fieldPatterns)
	{
		ptrArrayDelete (p->fieldPatterns);
		p->fieldPatterns = NULL;
	}

	eFree (p->pattern_string);

	if (p->message.message_string)
		eFree (p->message.message_string);

	if (p->anonymous_tag_prefix)
		eFree (p->anonymous_tag_prefix);

	eFree (p);
}

static void clearPatternSet (struct lregexControlBlock *lcb)
{
	ptrArrayClear (lcb->entries [REG_PARSER_SINGLE_LINE]);
	ptrArrayClear (lcb->entries [REG_PARSER_MULTI_LINE]);
	ptrArrayClear (lcb->tables);
}

extern struct lregexControlBlock* allocLregexControlBlock (parserDefinition *parser)
{
	struct lregexControlBlock *lcb = xCalloc (1, struct lregexControlBlock);

	lcb->entries[REG_PARSER_SINGLE_LINE] = ptrArrayNew(deleteTableEntry);
	lcb->entries[REG_PARSER_MULTI_LINE] = ptrArrayNew(deleteTableEntry);
	lcb->tables = ptrArrayNew(deleteTable);
	lcb->tstack = ptrArrayNew(NULL);
	lcb->guest_req = guestRequestNew ();
	lcb->owner = parser->id;

	return lcb;
}

extern void freeLregexControlBlock (struct lregexControlBlock* lcb)
{
	clearPatternSet (lcb);

	ptrArrayDelete (lcb->entries [REG_PARSER_SINGLE_LINE]);
	lcb->entries [REG_PARSER_SINGLE_LINE] = NULL;
	ptrArrayDelete (lcb->entries [REG_PARSER_MULTI_LINE]);
	lcb->entries [REG_PARSER_MULTI_LINE] = NULL;

	ptrArrayDelete (lcb->tables);
	lcb->tables = NULL;

	ptrArrayDelete (lcb->tstack);
	lcb->tstack = NULL;

	guestRequestDelete (lcb->guest_req);
	lcb->guest_req = NULL;

	eFree (lcb);
}

/*
*   Regex pseudo-parser
*/

static void initRegexTag (tagEntryInfo *e,
		const vString* const name, int kindIndex, int roleIndex, int scopeIndex, int placeholder,
		unsigned long line, MIOPos *pos, int xtag_type)
{
	Assert (name != NULL  &&  ((vStringLength (name) > 0) || placeholder));
	initRefTagEntry (e, vStringValue (name), kindIndex, roleIndex);
	e->extensionFields.scopeIndex = scopeIndex;
	e->placeholder = !!placeholder;
	if (line)
	{
		e->lineNumber = line;
		e->filePosition = *pos;
	}

	if (xtag_type != XTAG_UNKNOWN)
		markTagExtraBit (e, xtag_type);
}

/*
*   Regex pattern definition
*/

/* Take a string like "/blah/" and turn it into "blah", making sure
 * that the first and last characters are the same, and handling
 * quoted separator characters.  Actually, stops on the occurrence of
 * an unquoted separator.  Also turns "\t" into a Tab character.
 * Turns "\n" into a Newline character if MULTILINE is true.
 * Returns pointer to terminating separator.  Works in place.  Null
 * terminates name string.
 */
static char* scanSeparators (char* name, enum regexParserType regptype)
{
	char sep = name [0];
	char *copyto = name;
	bool quoted = false;

	for (++name ; *name != '\0' ; ++name)
	{
		if (quoted)
		{
			if (*name == sep)
				*copyto++ = sep;
			else if (*name == 't')
				*copyto++ = '\t';
			else if ((regptype == REG_PARSER_MULTI_LINE
					  || (regptype == REG_PARSER_MULTI_TABLE))
					 && *name == 'n')
				*copyto++ = '\n';
			else
			{
				/* Something else is quoted, so preserve the quote. */
				*copyto++ = '\\';
				*copyto++ = *name;
			}
			quoted = false;
		}
		else if (*name == '\\')
			quoted = true;
		else if (*name == sep)
		{
			break;
		}
		else
			*copyto++ = *name;
	}
	*copyto = '\0';
	return name;
}

/* Parse `regexp', in form "/regex/name/[k,Kind/]flags" (where the separator
 * character is whatever the first character of `regexp' is), by breaking it
 * up into null terminated strings, removing the separators, and expanding
 * '\t' into tabs. When complete, `regexp' points to the line matching
 * pattern, a pointer to the name matching pattern is written to `name', a
 * pointer to the kinds is written to `kinds' (possibly NULL), and a pointer
 * to the trailing flags is written to `flags'. If the pattern is not in the
 * correct format, a false value is returned.
 */
static bool parseTagRegex (
		enum regexParserType regptype,
		char* const regexp, char** const name,
		char** const kinds, char** const flags)
{
	bool result = false;
	const int separator = (unsigned char) regexp [0];

	*name = scanSeparators (regexp, regptype);
	if (*regexp == '\0')
		error (WARNING, "empty regexp");
	else if (**name != separator)
		error (WARNING, "%s: incomplete regexp", regexp);
	else
	{
		char* const third = scanSeparators (*name, false);
		if (**name != '\0' && (*name) [strlen (*name) - 1] == '\\')
			error (WARNING, "error in name pattern: \"%s\"", *name);
		if (*third != separator)
			error (WARNING, "%s: regexp missing final separator", regexp);
		else
		{
			/*
			 * first----------V third------------V
			 * --regex-<LANG>=/regexp/replacement/[kind-spec/][flags]
			 * second----------------^ fourth---------------^
			 */

			char* const fourth = scanSeparators (third, false);
			if (*fourth == separator)
			{
				*kinds = third;
				scanSeparators (fourth, false);
				*flags = fourth;
			}
			else
			{
				*flags = third;
				*kinds = NULL;
			}
			result = true;
		}
	}
	return result;
}


static void pre_ptrn_flag_exclusive_short (char c CTAGS_ATTR_UNUSED, void* data)
{
	bool *exclusive = data;
	*exclusive = true;
}

static void pre_ptrn_flag_exclusive_long (const char* const s CTAGS_ATTR_UNUSED, const char* const unused CTAGS_ATTR_UNUSED, void* data)
{
	pre_ptrn_flag_exclusive_short ('x', data);
}

static flagDefinition prePtrnFlagDef[] = {
	{ 'x',  "exclusive", pre_ptrn_flag_exclusive_short, pre_ptrn_flag_exclusive_long ,
	  NULL, "skip testing the other patterns if a line is matched to this pattern"},
};

static void scope_ptrn_flag_eval (const char* const f  CTAGS_ATTR_UNUSED,
				  const char* const v, void* data)
{
	unsigned int *bfields = data;

	if (strcmp (v, "ref") == 0)
		*bfields |= SCOPE_REF;
	else if (strcmp (v, "push") == 0)
		*bfields |= (SCOPE_PUSH | SCOPE_REF);
	else if (strcmp (v, "pop") == 0)
		*bfields |= SCOPE_POP;
	else if (strcmp (v, "clear") == 0)
		*bfields |= SCOPE_CLEAR;
	else if (strcmp (v, "set") == 0)
		*bfields |= (SCOPE_CLEAR | SCOPE_PUSH);
	else
		error (FATAL, "Unexpected value for scope flag in regex definition: scope=%s", v);
}

static void placeholder_ptrn_flag_eval (const char* const f  CTAGS_ATTR_UNUSED,
				     const char* const v  CTAGS_ATTR_UNUSED, void* data)
{
	unsigned int *bfields = data;
	*bfields |= SCOPE_PLACEHOLDER;
}

static flagDefinition scopePtrnFlagDef[] = {
	{ '\0', "scope",     NULL, scope_ptrn_flag_eval,
	  "ACTION", "use scope stack: ACTION = ref|push|pop|clear|set"},
	{ '\0', "placeholder",  NULL, placeholder_ptrn_flag_eval,
	  NULL, "don't put this tag to tags file."},
};

static kindDefinition *kindNew (char letter, const char *name, const char *description)
{
	kindDefinition *kdef = xCalloc (1, kindDefinition);
	kdef->letter        = letter;
	kdef->name = eStrdup (name);
	kdef->description = eStrdup(description? description: kdef->name);
	kdef->enabled = true;
	return kdef;
}

static void kindFree (kindDefinition *kind)
{
	kind->letter = '\0';
	eFree ((void *)kind->name);
	kind->name = NULL;
	eFree ((void *)kind->description);
	kind->description = NULL;
	eFree (kind);
}

static void initMgroup(struct mGroupSpec *mgroup)
{
	mgroup->forLineNumberDetermination = NO_MULTILINE;
	mgroup->forNextScanning = NO_MULTILINE;
	mgroup->nextFromStart = false;
}

static void initGuestSpec (struct guestSpec *guest)
{
	guest->lang.type = GUEST_LANG_UNKNOWN;
}

static void initTaction(struct mTableActionSpec *taction)
{
	taction->action = TACTION_NOP;
	taction->table = NULL;
}

static regexPattern * refPattern (regexPattern * ptrn)
{
	ptrn->refcount++;
	return ptrn;
}

static regexPattern * newPattern (regex_t* const pattern,
								  enum regexParserType regptype)
{
	regexPattern *ptrn = xCalloc(1, regexPattern);

	ptrn->pattern = pattern;
	ptrn->exclusive = false;
	ptrn->accept_empty_name = false;
	ptrn->regptype = regptype;
	ptrn->xtagType = XTAG_UNKNOWN;

	if (regptype == REG_PARSER_MULTI_LINE)
		initMgroup(&ptrn->mgroup);
	if (regptype == REG_PARSER_MULTI_TABLE)
		initTaction(&ptrn->taction);
	initGuestSpec (&ptrn->guest);

	ptrn->u.tag.roleBits = 0;
	ptrn->refcount = 1;
	return ptrn;
}

static regexTableEntry * newRefPatternEntry (regexTableEntry * other)
{
	regexTableEntry *entry = xCalloc (1, regexTableEntry);

	Assert (other && other->pattern);

	entry->pattern = refPattern(other->pattern);
	return entry;
}

static regexTableEntry * newEntry (regex_t* const pattern,
								   enum regexParserType regptype)
{
	regexTableEntry *entry = xCalloc (1, regexTableEntry);
	entry->pattern = newPattern (pattern, regptype);
	return entry;
}

static regexPattern* addCompiledTagCommon (struct lregexControlBlock *lcb,
										   int table_index,
										   regex_t* const pattern,
										   enum regexParserType regptype)
{
	regexTableEntry *entry = newEntry (pattern, regptype);

	if (regptype == REG_PARSER_MULTI_TABLE)
	{
		struct regexTable *table = ptrArrayItem (lcb->tables, table_index);
		Assert(table);

		ptrArrayAdd (table->entries, entry);
	}
	else
		ptrArrayAdd (lcb->entries[regptype], entry);

	useRegexMethod(lcb->owner);

	return entry->pattern;
}

static void pre_ptrn_flag_mgroup_long (const char* const s, const char* const v, void* data)
{
	struct mGroupSpec *mgroup = data;
	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}
	if (!strToInt (v, 10, &mgroup->forLineNumberDetermination))
	{
		error (WARNING, "wrong %s specification: %s", s, v);
		mgroup->forLineNumberDetermination = NO_MULTILINE;
	}
	else if (mgroup->forLineNumberDetermination < 0
			 || mgroup->forLineNumberDetermination >= BACK_REFERENCE_COUNT)
	{
		error (WARNING, "out of range(0 ~ %d) %s specification: %s",
			   (BACK_REFERENCE_COUNT - 1),
			   s, v);
		mgroup->forLineNumberDetermination = NO_MULTILINE;
	}
}

static void pre_ptrn_flag_advanceTo_long (const char* const s, const char* const v, void* data)
{
	struct mGroupSpec *mgroup = data;
	char *vdup;
	char *tmp;


	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}

	vdup = eStrdup (v);

	mgroup->nextFromStart = false;
	if ((tmp = strstr(vdup, "start")))
	{
		mgroup->nextFromStart = true;
		*tmp = '\0';
	}
	else if ((tmp = strstr(vdup, "end")))
		*tmp = '\0';

	if (!strToInt (vdup, 10, &(mgroup->forNextScanning)))
	{
		error (WARNING, "wrong %s specification: %s", s, vdup);
		mgroup->nextFromStart = false;
	}
	else if (mgroup->forNextScanning < 0 || mgroup->forNextScanning >= BACK_REFERENCE_COUNT)
	{
		error (WARNING, "out of range(0 ~ %d) %s specification: %s",
			   (BACK_REFERENCE_COUNT - 1), s, vdup);
		mgroup->nextFromStart = false;
	}

	eFree (vdup);
}

struct guestPtrnFlagData {
	enum regexParserType type;
	struct guestSpec *guest;
};

static void pre_ptrn_flag_guest_long (const char* const s, const char* const v, void* data)
{
	struct guestPtrnFlagData *flagData = data;
	enum regexParserType type = flagData->type;
	struct guestSpec *guest = flagData->guest;
	struct boundarySpec *current;

	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}

	char *tmp = strchr (v, ',');
	if (tmp == NULL)
	{
		error (WARNING, "no terminator found for parser name: %s", s);
		return;
	}

	if ((tmp - v) == 0)
	{
		if (type == REG_PARSER_MULTI_LINE)
		{
			error (WARNING,
				   "using placeholder for guest name field is not allowed in multiline regex spec: %s", v);
			goto err;
		}

		guest->lang.type = GUEST_LANG_PLACEHOLDER;
	}
	else if (*v == '\\' || *v == '*')
	{
		const char *n_tmp = v + 1;
		const char *n = n_tmp;
		for (; isdigit (*n_tmp); n_tmp++);
		char c = *n_tmp;
		*(char *)n_tmp = '\0';
		if (!strToInt (n, 10, &(guest->lang.spec.patternGroup)))
		{
			error (WARNING, "wrong guest name specification: %s", v);
			goto err;
		}
		else if (guest->lang.spec.patternGroup >= BACK_REFERENCE_COUNT)
		{
			error (WARNING, "wrong guest name specification (back reference count is too large): %d",
				   guest->lang.spec.patternGroup);
			goto err;
		}

		*(char *)n_tmp = c;
		if (*n_tmp != ',')
		{
			error (WARNING, "wrong guest specification (garbage at the end of end guest spec): %s", v);
			goto err;
		}

		guest->lang.type = (*v == '\\')
			? GUEST_LANG_PTN_GROUP_FOR_LANGNAME
			: GUEST_LANG_PTN_GROUP_FOR_FILEMAP;
	}
	else
	{
		guest->lang.spec.lang = getNamedLanguage (v, (tmp - v));
		if (guest->lang.spec.lang == LANG_IGNORE)
		{
			error (WARNING, "no parser found for the guest spec: %s", v);
			goto err;
		}
		guest->lang.type = GUEST_LANG_STATIC_LANGNAME;
	}

	tmp++;
	if (*tmp == '\0')
	{
		error (WARNING, "no area spec found in the guest spec: %s", v);
		goto err;
	}

	for (int i = 0; i < 2; i++)
	{
		current = guest->boundary + i;
		const char *current_field_str = (i == BOUNDARY_START? "start": "end");

		if (tmp [0] == ((i == BOUNDARY_START)? ',': '\0'))
		{
			if (type == REG_PARSER_MULTI_LINE)
				error (WARNING,
					   "using placeholder for %s field is not allowed in multiline regex spec: %s",
					   current_field_str, v);

			current->placeholder = true;
		}
		else
		{
			char *n = tmp;

			for (; isdigit (*tmp); tmp++);
			char c = *tmp;
			*tmp = '\0';
			if (!strToInt (n, 10, &(current->patternGroup)))
			{
				error (WARNING, "wrong guest area specification (patternGroup of %s, number expected): %s:%s",
					   current_field_str, v, n);
				goto err;
			}
			*tmp = c;
			if (*tmp == '\0')
			{
				error (WARNING, "wrong guest area specification (patternGroup of %s, nether start nor end given): %s",
					   current_field_str, v);
				goto err;
			}
			else if (strncmp (tmp, "start", 5) == 0)
			{
				current->fromStartOfGroup = true;
				tmp += 5;
			}
			else if (strncmp (tmp, "end", 3) == 0)
			{
				current->fromStartOfGroup = false;
				tmp += 3;
			}
			else
			{
				error (WARNING, "wrong guest area specification (%s): %s",
					   current_field_str, v);
				goto err;
			}
		}

		if (i == 0)
		{
			if (*tmp != ',')
			{
				error (WARNING,
					   "wrong guest area specification (separator between start and end boundaries): %s", v);
				goto err;
			}
			tmp++;
		}
		else if (i == 1 && (*tmp != '\0'))
		{
			error (WARNING, "wrong guest area specification (garbage at the end of end boundary spec): %s", v);
			goto err;
		}
	}
	return;
 err:
	guest->lang.type = GUEST_LANG_UNKNOWN;
}

static flagDefinition multilinePtrnFlagDef[] = {
	{ '\0',  "mgroup", NULL, pre_ptrn_flag_mgroup_long,
	  "N", "a group in pattern determining the line number of tag"},
	{ '\0',  "_advanceTo", NULL, pre_ptrn_flag_advanceTo_long,
	  "N[start|end]", "a group in pattern from where the next scan starts [0end]"},
};

static flagDefinition guestPtrnFlagDef[] = {
#define EXPERIMENTAL "_"
	{ '\0',  EXPERIMENTAL "guest", NULL, pre_ptrn_flag_guest_long,
	  "PARSERSPEC,N0[start|end],N1[start|end]", "run guest parser on the area"},
};

static bool hasMessage(const regexPattern *const ptrn)
{
	return (ptrn->message.selection > 0 && ptrn->message.message_string);
}

struct commonFlagData {
	const langType owner;
	const struct lregexControlBlock *const lcb;
	regexPattern *ptrn;
};

static void common_flag_msg_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData *cdata = data;
	regexPattern *ptrn = cdata->ptrn;

	Assert (ptrn);

	if (hasMessage(ptrn))
	{
		error (WARNING, "only one message flag may be given per regex (already set to '%s')",
			   ptrn->message.message_string);
		return;
	}

	if (strcmp (s, "fatal") == 0)
	{
		ptrn->message.selection = FATAL;
	}
	else if (strcmp (s, "warning") == 0)
	{
		ptrn->message.selection = WARNING;
	}

	Assert (ptrn->message.selection != 0);

	if (!v || !*v)
	{
		error (WARNING, "no message value is given for {%s}", s);
		return;
	}

	const char* begin = v;
	const char* end = v + strlen (v);
	--end;

	if (*begin != '"' || *end != '"' || begin == end)
	{
		error (WARNING, "argument for {%s} must be in double-quotes", s);
		return;
	}

	++begin;

	if (begin < end)
		ptrn->message.message_string = eStrndup (begin, end - begin);
}

static void common_flag_extra_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData * cdata = data;

	Assert (cdata->ptrn);

	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}

	cdata->ptrn->xtagType = getXtagTypeForNameAndLanguage (v, cdata->owner);
	if (cdata->ptrn->xtagType == XTAG_UNKNOWN)
		error (WARNING, "no such extra \"%s\" in %s", v, getLanguageName(cdata->owner));
}


static struct fieldPattern * fieldPatternNew (fieldType ftype, const char *template)
{
	struct fieldPattern *fp;

	fp = xMalloc(1, struct fieldPattern);
	fp->ftype = ftype;
	fp->template = eStrdup(template);

	return fp;
}

static void fieldPatternDelete (struct fieldPattern *fp)
{
	eFree ((void *)fp->template);
	eFree (fp);
}

static void common_flag_field_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData * cdata = data;
	regexPattern *ptrn = cdata->ptrn;

	Assert (ptrn);

	struct fieldPattern *fp;
	fieldType ftype;
	char *fname;
	const char* template;
	char *tmp;

	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}

	tmp = strchr (v, ':');
	if (tmp == NULL || tmp == v)
	{
		error (WARNING, "no field name is given for: %s", s);
		return;
	}

	fname = eStrndup (v, tmp - v);
	ftype = getFieldTypeForNameAndLanguage (fname, cdata->owner);
	if (ftype == FIELD_UNKNOWN)
	{
		error (WARNING, "no such field \"%s\" in %s", fname, getLanguageName(cdata->owner));
		eFree (fname);
		return;
	}

	if (ptrn->fieldPatterns)
	{
		for (unsigned int i = 0; i < ptrArrayCount(ptrn->fieldPatterns); i++)
		{
			fp = ptrArrayItem(ptrn->fieldPatterns, i);
			if (fp->ftype == ftype)
			{
				error (WARNING, "duplicated field specification \"%s\" in %s", fname, getLanguageName(cdata->owner));
				eFree (fname);
				return;
			}
		}
	}
	eFree (fname);

	template = tmp + 1;
	fp = fieldPatternNew (ftype, template);

	if (ptrn->fieldPatterns == NULL)
		ptrn->fieldPatterns = ptrArrayNew((ptrArrayDeleteFunc)fieldPatternDelete);
	ptrArrayAdd(ptrn->fieldPatterns, fp);
}

static void common_flag_role_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData * cdata = data;
	regexPattern *ptrn = cdata->ptrn;
	roleDefinition * role;

	Assert (ptrn);

	if (!v)
	{
		error (WARNING, "no value is given for: %s", s);
		return;
	}

	role = getLanguageRoleForName(cdata->owner,
								  ptrn->u.tag.kindIndex, v);
	if (!role)
	{
		error (WARNING, "no such role: %s", v);
		return;
	}

	ptrn->u.tag.roleBits |= makeRoleBit(role->id);
}

static void common_flag_anonymous_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData * cdata = data;
	regexPattern *ptrn = cdata->ptrn;

	Assert (ptrn);

	if (ptrn->anonymous_tag_prefix)
	{
		error (WARNING, "an anonymous tag prefix for this pattern (%s) is already given: %s",
			   ptrn->pattern_string? ptrn->pattern_string: "",
			   ptrn->anonymous_tag_prefix);
		return;
	}

	if (!v)
	{
		error (WARNING, "no PREFIX for anonymous regex flag is given (pattern == %s)",
			   ptrn->pattern_string? ptrn->pattern_string: "");
		return;
	}

	if (ptrn->u.tag.kindIndex == KIND_GHOST_INDEX)
	{
		error (WARNING, "use \"%s\" regex flag only with an explicitly defined kind", s);
		return;
	}

	ptrn->anonymous_tag_prefix = eStrdup (v);
}

static flagDefinition commonSpecFlagDef[] = {
	{ '\0',  "fatal", NULL, common_flag_msg_long ,
	  "\"MESSAGE\"", "print the given MESSAGE and exit"},
	{ '\0',  "warning", NULL, common_flag_msg_long ,
	  "\"MESSAGE\"", "print the given MESSAGE at WARNING level"},
#define EXPERIMENTAL "_"
	{ '\0',  EXPERIMENTAL "extra", NULL, common_flag_extra_long ,
	  "EXTRA", "record the tag only when the extra is enabled"},
	{ '\0',  EXPERIMENTAL "field", NULL, common_flag_field_long ,
	  "FIELD:VALUE", "record the matched string(VALUE) to parser own FIELD of the tag"},
	{ '\0',  EXPERIMENTAL "role", NULL, common_flag_role_long,
	  "ROLE", "set the given ROLE to the roles field"},
	{ '\0',  EXPERIMENTAL "anonymous", NULL, common_flag_anonymous_long,
	  "PREFIX", "make an anonymous tag with PREFIX"},
};


static void pre_ptrn_flag_mtable_long (const char* const s, const char* const v, void* data)
{
	struct commonFlagData * cdata = data;
	regexPattern *ptrn = cdata->ptrn;
	struct mTableActionSpec *taction;
	bool taking_table = true;

	Assert (ptrn);
	Assert (cdata->lcb);

	taction = &ptrn->taction;

	if (strcmp (s, "tenter") == 0)
		taction->action = TACTION_ENTER;
	else if (strcmp (s, "tleave") == 0)
	{
		taction->action = TACTION_LEAVE;
		taking_table = false;
	}
	else if (strcmp (s, "tjump") == 0)
		taction->action = TACTION_JUMP;
	else if (strcmp (s, "treset") == 0)
		taction->action = TACTION_RESET;
	else if (strcmp (s, "tquit") == 0)
	{
		taction->action = TACTION_QUIT;
		taking_table = false;
	}

	if (taking_table)
	{
		int t;
		char *continuation = NULL;


		if (!v || (!*v))
			error (FATAL, "no table is given for table action: %s", s);

		if (taction->action == TACTION_ENTER
			&& (continuation = strchr (v, ',')))
		{
			char *tableEnterTo;

			tableEnterTo = eStrndup (v, continuation - v);
			t = getTableIndexForName (cdata->lcb, tableEnterTo);
			if (t < 0)
				error (FATAL, "table is not defined: %s", tableEnterTo);
			taction->table = ptrArrayItem (cdata->lcb->tables, t);
			eFree (tableEnterTo);

			if (!*(continuation + 1))
				error (FATAL, "no continuation table is given for: %s", v);

			int t_cont = getTableIndexForName (cdata->lcb, continuation + 1);
			if (t_cont < 0)
				error (FATAL, "table for continuation is not defined: %s", continuation + 1);
			taction->continuation_table = ptrArrayItem (cdata->lcb->tables, t_cont);
		}
		else
		{
			t = getTableIndexForName (cdata->lcb, v);
			if (t < 0)
				error (FATAL, "table is not defined: %s", v);
			taction->table = ptrArrayItem (cdata->lcb->tables, t);
			taction->continuation_table = NULL;
		}
	}
}

static flagDefinition multitablePtrnFlagDef[] = {
	{ '\0',  "tenter", NULL, pre_ptrn_flag_mtable_long ,
	  "TABLE[,CONT]", "enter to given regext table (with specifying continuation)"},
	{ '\0',  "tleave", NULL, pre_ptrn_flag_mtable_long ,
	  NULL, "leave from the current regext table"},
	{ '\0',  "tjump", NULL, pre_ptrn_flag_mtable_long ,
	  "TABLE", "jump to another regext table(don't push the current table to state stack)"},
	{ '\0',  "treset", NULL, pre_ptrn_flag_mtable_long ,
	  "TABLE", "clear the state stack and jump to given regex table"},
	{ '\0',  "tquit", NULL, pre_ptrn_flag_mtable_long ,
	  NULL, "stop the parsing with this parser"},
};


static void setKind(regexPattern * ptrn, const langType owner,
					const char kindLetter, const char* kindName,
					const char *const description,
					bool kind_explicitly_defined)
{
	Assert (ptrn);
	Assert (ptrn->u.tag.name_pattern);
	Assert (kindName);
	kindDefinition *kdef = getLanguageKindForLetter (owner, kindLetter);

	if (kdef)
	{
		if (strcmp (kdef->name, kindName) && (strcmp(kindName, KIND_REGEX_DEFAULT_NAME)))
			/* When using a same kind letter for multiple regex patterns, the name of kind
			   should be the same. */
			error  (WARNING, "Don't reuse the kind letter `%c' in a language %s (old: \"%s\", new: \"%s\")",
					kdef->letter, getLanguageName (owner),
					kdef->name, kindName);
		ptrn->u.tag.kindIndex = kdef->id;
	}
	else if (*ptrn->u.tag.name_pattern == '\0' &&
			 kindLetter == KIND_REGEX_DEFAULT_LETTER &&
			 (strcmp(kindName, KIND_REGEX_DEFAULT_NAME) == 0) &&
			 (!kind_explicitly_defined))
		ptrn->u.tag.kindIndex = KIND_GHOST_INDEX;
	else
	{
		kdef = kindNew (kindLetter, kindName, description);
		defineLanguageKind (owner, kdef, kindFree);
		ptrn->u.tag.kindIndex = kdef->id;
	}
}

static void patternEvalFlags (struct lregexControlBlock *lcb,
							  regexPattern * ptrn,
							  enum regexParserType regptype,
							  const char* flags)
{
	struct commonFlagData commonFlagData = {
		.owner = lcb->owner,
		.lcb = lcb,
		.ptrn = ptrn
	};

	if (regptype == REG_PARSER_SINGLE_LINE)
		flagsEval (flags, prePtrnFlagDef, ARRAY_SIZE(prePtrnFlagDef), &ptrn->exclusive);

	flagsEval (flags, commonSpecFlagDef, ARRAY_SIZE(commonSpecFlagDef), &commonFlagData);

	if (regptype == REG_PARSER_SINGLE_LINE || regptype == REG_PARSER_MULTI_TABLE)
		flagsEval (flags, scopePtrnFlagDef, ARRAY_SIZE(scopePtrnFlagDef), &ptrn->scopeActions);

	if (regptype == REG_PARSER_MULTI_LINE || regptype == REG_PARSER_MULTI_TABLE)
	{
		ptrn->mgroup.forNextScanning = 0;
		/* ptrn->mgroup.nextFromStart is initialized in initMgroup() already. */
		flagsEval (flags, multilinePtrnFlagDef, ARRAY_SIZE(multilinePtrnFlagDef), &ptrn->mgroup);
	}

	struct guestPtrnFlagData guestPtrnFlagData = {
		.type = regptype,
		.guest = &ptrn->guest,
	};
	flagsEval (flags, guestPtrnFlagDef, ARRAY_SIZE(guestPtrnFlagDef), &guestPtrnFlagData);

	if (regptype == REG_PARSER_MULTI_TABLE)
		flagsEval (flags, multitablePtrnFlagDef, ARRAY_SIZE(multitablePtrnFlagDef), &commonFlagData);
}

static regexPattern *addCompiledTagPattern (struct lregexControlBlock *lcb,
											int table_index,
											enum regexParserType regptype, regex_t* const pattern,
					    const char* const name, char kindLetter, const char* kindName,
					    char *const description, const char* flags,
					    bool kind_explicitly_defined,
					    bool *disabled)
{
	regexPattern * ptrn = addCompiledTagCommon(lcb, table_index, pattern, regptype);

	ptrn->type = PTRN_TAG;
	ptrn->u.tag.name_pattern = eStrdup (name);
	ptrn->disabled = disabled;

	setKind(ptrn, lcb->owner, kindLetter, kindName, description, kind_explicitly_defined);
	patternEvalFlags (lcb, ptrn, regptype, flags);

	return ptrn;
}

static regexPattern *addCompiledCallbackPattern (struct lregexControlBlock *lcb, regex_t* const pattern,
					const regexCallback callback, const char* flags,
					bool *disabled,
					void *userData)
{
	regexPattern * ptrn;
	bool exclusive = false;
	flagsEval (flags, prePtrnFlagDef, ARRAY_SIZE(prePtrnFlagDef), &exclusive);
	ptrn = addCompiledTagCommon(lcb, TABLE_INDEX_UNUSED, pattern, REG_PARSER_SINGLE_LINE);
	ptrn->type    = PTRN_CALLBACK;
	ptrn->u.callback.function = callback;
	ptrn->u.callback.userData = userData;
	ptrn->exclusive = exclusive;
	ptrn->disabled = disabled;
	return ptrn;
}


static void regex_flag_basic_short (char c CTAGS_ATTR_UNUSED, void* data)
{
	int* cflags = data;
	*cflags &= ~REG_EXTENDED;
}

static void regex_flag_basic_long (const char* const s CTAGS_ATTR_UNUSED, const char* const unused CTAGS_ATTR_UNUSED, void* data)
{
	regex_flag_basic_short ('b', data);
}

static void regex_flag_extend_short (char c CTAGS_ATTR_UNUSED, void* data)
{
	int* cflags = data;
	*cflags |= REG_EXTENDED;
}

static void regex_flag_extend_long (const char* const c CTAGS_ATTR_UNUSED, const char* const unused CTAGS_ATTR_UNUSED, void* data)
{
	regex_flag_extend_short('e', data);
}

static void regex_flag_icase_short (char c CTAGS_ATTR_UNUSED, void* data)
{
	int* cflags = data;
	*cflags |= REG_ICASE;
}

static void regex_flag_icase_long (const char* s CTAGS_ATTR_UNUSED, const char* const unused CTAGS_ATTR_UNUSED, void* data)
{
	regex_flag_icase_short ('i', data);
}


static flagDefinition regexFlagDefs[] = {
	{ 'b', "basic",  regex_flag_basic_short,  regex_flag_basic_long,
	  NULL, "interpreted as a Posix basic regular expression."},
	{ 'e', "extend", regex_flag_extend_short, regex_flag_extend_long,
	  NULL, "interpreted as a Posix extended regular expression (default)"},
	{ 'i', "icase",  regex_flag_icase_short,  regex_flag_icase_long,
	  NULL, "applied in a case-insensitive manner"},
};

static regex_t* compileRegex (enum regexParserType regptype,
							  const char* const regexp, const char* const flags)
{
	int cflags = REG_EXTENDED | REG_NEWLINE;

	if (regptype == REG_PARSER_MULTI_TABLE)
		cflags &= ~REG_NEWLINE;

	regex_t *result;
	int errcode;

	flagsEval (flags,
		   regexFlagDefs,
		   ARRAY_SIZE(regexFlagDefs),
		   &cflags);

	result = xMalloc (1, regex_t);
	errcode = regcomp (result, regexp, cflags);
	if (errcode != 0)
	{
		char errmsg[256];
		regerror (errcode, result, errmsg, 256);
		error (WARNING, "regcomp %s: %s", regexp, errmsg);
		regfree (result);
		eFree (result);
		result = NULL;
	}
	return result;
}


/* If a letter and/or a name are defined in kindSpec, return true. */
static bool parseKinds (
		const char* const kindSpec, char* const kindLetter, char** const kindName,
		char **description)
{
	*description = NULL;

	if (kindSpec == NULL  ||  kindSpec [0] == '\0')
	{
		*kindLetter = KIND_REGEX_DEFAULT_LETTER;
		*kindName = eStrdup (KIND_REGEX_DEFAULT_NAME);
		return false;
	}
	else
	{
		bool explicitly_defined = false;
		const char* k = kindSpec;

		if (k [0] != ','  &&  (k [1] == ','  ||  k [1] == '\0'))
		{
			*kindLetter = *k++;
			explicitly_defined = true;
		}
		else
			*kindLetter = KIND_REGEX_DEFAULT_LETTER;

		if (*k == ',')
			++k;

		if (k [0] == '\0')
			*kindName = eStrdup (KIND_REGEX_DEFAULT_NAME);
		else
		{
			const char *const comma = strchr (k, ',');

			if (comma == NULL)
			{
				if (strlen (k) == 0)
					*kindName = eStrdup (KIND_REGEX_DEFAULT_NAME);
				else
				{
					*kindName = eStrdup (k);
					explicitly_defined = true;
				}
			}
			else
			{
				if (comma - k == 0)
					*kindName = eStrdup (KIND_REGEX_DEFAULT_NAME);
				else
				{
					*kindName = eStrndup (k, comma - k );
					explicitly_defined = true;
				}
				k = comma + 1;
				if (k [0] != '\0')
					*description = eStrdup (k);
			}
		}
		return explicitly_defined;
	}
}

/*
*   Regex pattern matching
*/


static vString* substitute (
		const char* const in, const char* out,
		const int nmatch, const regmatch_t* const pmatch)
{
	vString* result = vStringNew ();
	const char* p;
	for (p = out  ;  *p != '\0'  ;  p++)
	{
		if (*p == '\\'  &&  isdigit ((int) *++p))
		{
			const int dig = *p - '0';
			if (0 < dig  &&  dig < nmatch  &&  pmatch [dig].rm_so != -1)
			{
				const int diglen = pmatch [dig].rm_eo - pmatch [dig].rm_so;
				vStringNCatS (result, in + pmatch [dig].rm_so, diglen);
			}
		}
		else if (*p != '\n'  &&  *p != '\r')
			vStringPut (result, *p);
	}
	return result;
}

static unsigned long getInputLineNumberInRegPType (enum regexParserType regptype,
												   off_t offset)
{
	return (regptype == REG_PARSER_MULTI_LINE || regptype == REG_PARSER_MULTI_TABLE)
		? getInputLineNumberForFileOffset (offset)
		: getInputLineNumber ();
}

static void fillEndLineFieldOfUpperScopes (struct lregexControlBlock *lcb, unsigned long endline)
{
	tagEntryInfo *entry;
	int n = lcb->currentScope;

	while ((entry = getEntryInCorkQueue (n))
		   && (entry->extensionFields.endLine == 0))
	{
		entry->extensionFields.endLine = endline;
		n = entry->extensionFields.scopeIndex;
	}
}

static void matchTagPattern (struct lregexControlBlock *lcb,
		const char* line,
		const regexPattern* const patbuf,
		const regmatch_t* const pmatch,
			     off_t offset)
{
	vString *const name =
		(patbuf->u.tag.name_pattern[0] != '\0') ? substitute (line,
															  patbuf->u.tag.name_pattern,
															  BACK_REFERENCE_COUNT, pmatch):
		(patbuf->anonymous_tag_prefix) ? anonGenerateNew (patbuf->anonymous_tag_prefix,
														  patbuf->u.tag.kindIndex):
		vStringNewInit ("");
	bool placeholder = !!((patbuf->scopeActions & SCOPE_PLACEHOLDER) == SCOPE_PLACEHOLDER);
	int scope = CORK_NIL;
	int n;

	vStringStripLeading (name);
	vStringStripTrailing (name);

	if (patbuf->scopeActions & SCOPE_REF)
	{
		tagEntryInfo *entry;

		scope = lcb->currentScope;
		while ((entry = getEntryInCorkQueue (scope)) && entry->placeholder)
			/* Look at parent */
			scope = entry->extensionFields.scopeIndex;
	}
	if (patbuf->scopeActions & SCOPE_CLEAR)
	{
		unsigned long endline = getInputLineNumberInRegPType(patbuf->regptype, offset);
		fillEndLineFieldOfUpperScopes (lcb, endline);
		lcb->currentScope = CORK_NIL;
	}
	if (patbuf->scopeActions & SCOPE_POP)
	{
		tagEntryInfo *entry = getEntryInCorkQueue (lcb->currentScope);

		if (entry && (entry->extensionFields.endLine == 0))
			entry->extensionFields.endLine = getInputLineNumberInRegPType(patbuf->regptype, offset);

		lcb->currentScope = entry? entry->extensionFields.scopeIndex: CORK_NIL;
	}

	if (vStringLength (name) == 0 && (placeholder == false))
	{
		if (patbuf->accept_empty_name == false)
			error (WARNING, "%s:%lu: null expansion of name pattern \"%s\"",
			       getInputFileName (),
				   getInputLineNumberInRegPType(patbuf->regptype, offset),
			       patbuf->u.tag.name_pattern);
		n = CORK_NIL;
	}
	else
	{
		static TrashBox* field_trashbox;
		unsigned long ln = 0;
		MIOPos pos;
		tagEntryInfo e;
		int kind;
		roleBitsType roleBits;

		if ((patbuf->regptype == REG_PARSER_MULTI_LINE)
			|| (patbuf->regptype == REG_PARSER_MULTI_TABLE))
		{
			ln = getInputLineNumberForFileOffset (offset);
			pos = getInputFilePositionForLine (ln);
		}

		n = CORK_NIL;
		kind = patbuf->u.tag.kindIndex;
		roleBits = patbuf->u.tag.roleBits;

		initRegexTag (&e, name, kind, ROLE_DEFINITION_INDEX, scope, placeholder,
					  ln, ln == 0? NULL: &pos, patbuf->xtagType);

		if (field_trashbox == NULL)
		{
			field_trashbox = trashBoxNew();
			DEFAULT_TRASH_BOX (field_trashbox, trashBoxDelete);
		}

		if (patbuf->fieldPatterns)
		{
			for (unsigned int i = 0; i < ptrArrayCount(patbuf->fieldPatterns); i++)
			{
				struct fieldPattern *fp = ptrArrayItem(patbuf->fieldPatterns, i);
				if (isFieldEnabled (fp->ftype))
				{
					vString * const value = substitute (line, fp->template,
														BACK_REFERENCE_COUNT, pmatch);
					attachParserField (&e, false, fp->ftype, vStringValue (value));
					trashBoxPut (field_trashbox, value,
								 (TrashBoxDestroyItemProc)vStringDelete);
				}
			}
		}

		if (roleBits)
		{
			unsigned int roleIndex;

			for (roleIndex = 0;
				 roleIndex < countLanguageRoles(e.langType, kind);
				 roleIndex++)
			{
				if (roleBits & makeRoleBit(roleIndex))
					assignRole (&e, roleIndex);
			}
		}

		if (patbuf->anonymous_tag_prefix)
			markTagExtraBit (&e, XTAG_ANONYMOUS);

		n = makeTagEntry (&e);

		trashBoxMakeEmpty(field_trashbox);
	}

	if (patbuf->scopeActions & SCOPE_PUSH)
		lcb->currentScope = n;

	vStringDelete (name);
}

static bool matchCallbackPattern (
		const vString* const line, const regexPattern* const patbuf,
		const regmatch_t* const pmatch)
{
	regexMatch matches [BACK_REFERENCE_COUNT];
	unsigned int count = 0;
	int i;
	for (i = 0  ;  i < BACK_REFERENCE_COUNT  ;  ++i)
	{
		matches [i].start  = pmatch [i].rm_so;
		matches [i].length = pmatch [i].rm_eo - pmatch [i].rm_so;
		/* a valid match may have both offsets == -1,
		 * e.g. (foo)*(bar) matching "bar" - see CTags bug 271.
		 * As POSIX regex doesn't seem to have a way to count matches,
		 * we return the count up to the last non-empty match. */
		if (pmatch [i].rm_so != -1)
			count = i + 1;
	}
	return patbuf->u.callback.function (vStringValue (line), matches, count,
				     patbuf->u.callback.userData);
}


static void printMessage(const langType language,
						 const regexPattern *const ptrn,
						 const off_t offset,
						 const char *const line,
						 const regmatch_t* const pmatch)
{
	vString *msg;

	Assert (ptrn);
	Assert (ptrn->message.selection > 0);
	Assert (ptrn->message.message_string);

	msg = substitute (line, ptrn->message.message_string, BACK_REFERENCE_COUNT, pmatch);

	error (ptrn->message.selection, "%sMessage from regex<%s>: %s (%s:%lu)",
		   (ptrn->message.selection == FATAL ? "Fatal: " : ""),
		   getLanguageName (language),
		   vStringValue (msg),
		   getInputFileName (),
		   getInputLineNumberInRegPType (ptrn->regptype, offset));

	vStringDelete (msg);
}

static bool isGuestRequestConsistent (struct guestRequest *guest_req)
{
	return (guest_req->lang != LANG_IGNORE)
		&& (guest_req->boundary[BOUNDARY_START].offset < guest_req->boundary[BOUNDARY_END].offset);
}

static bool fillGuestRequest (const char *start,
							  const char *current,
							  regmatch_t pmatch [BACK_REFERENCE_COUNT],
							  struct guestSpec *guest_spec,
							  struct guestRequest *guest_req)
{
	if (guest_spec->lang.type == GUEST_LANG_UNKNOWN)
		return false;
	else if (guest_spec->lang.type == GUEST_LANG_PLACEHOLDER)
		;
	else if (guest_spec->lang.type == GUEST_LANG_STATIC_LANGNAME)
	{
		guest_req->lang = guest_spec->lang.spec.lang;
		guest_req->lang_set = true;
	}
	else if (guest_spec->lang.type == GUEST_LANG_PTN_GROUP_FOR_LANGNAME)
	{
		const char * name = current + pmatch [guest_spec->lang.spec.patternGroup].rm_so;
		int size = pmatch [guest_spec->lang.spec.patternGroup].rm_eo
			- pmatch [guest_spec->lang.spec.patternGroup].rm_so;
		if (size > 0)
		{
			guest_req->lang = getNamedLanguage (name, size);
			guest_req->lang_set = true;
		}
	}
	else if (guest_spec->lang.type == GUEST_LANG_PTN_GROUP_FOR_FILEMAP)
	{
		const char * name = current + pmatch [guest_spec->lang.spec.patternGroup].rm_so;
		int size = pmatch [guest_spec->lang.spec.patternGroup].rm_eo
			- pmatch [guest_spec->lang.spec.patternGroup].rm_so;
		char *fname = (size > 0)? eStrndup (name, size): NULL;

		if (fname)
		{
			guest_req->lang = getLanguageForFilename (fname, LANG_AUTO);
			guest_req->lang_set = true;
			eFree (fname);
		}
	}

	for (int i = 0; i < 2; i++)
	{
		struct boundarySpec *boundary_spec = guest_spec->boundary + i;
		struct boundaryInRequest *boundary = guest_req->boundary + i;
		if (!boundary_spec->placeholder)
		{
			boundary->offset =  current - start + (boundary_spec->fromStartOfGroup
												   ? pmatch [boundary_spec->patternGroup].rm_so
												   : pmatch [boundary_spec->patternGroup].rm_eo);
			boundary->offset_set = true;
		}
	}
	return guestRequestIsFilled (guest_req);
}

static bool matchRegexPattern (struct lregexControlBlock *lcb,
							   const vString* const line,
							   regexTableEntry *entry)
{
	bool result = false;
	regmatch_t pmatch [BACK_REFERENCE_COUNT];
	int match;
	regexPattern* patbuf = entry->pattern;
	struct guestSpec  *guest = &patbuf->guest;

	if (patbuf->disabled && *(patbuf->disabled))
		return false;

	match = regexec (patbuf->pattern, vStringValue (line),
			 BACK_REFERENCE_COUNT, pmatch, 0);
	if (match == 0)
	{
		result = true;
		entry->statistics.match++;

		if (hasMessage(patbuf))
			printMessage(lcb->owner, patbuf, 0, vStringValue (line), pmatch);

		if (patbuf->type == PTRN_TAG)
		{
			matchTagPattern (lcb, vStringValue (line), patbuf, pmatch, 0);

			if (guest->lang.type != GUEST_LANG_UNKNOWN)
			{
				unsigned long ln = getInputLineNumber ();
				long current = getInputFileOffsetForLine (ln);
				if (fillGuestRequest (vStringValue (line) - current,
									  vStringValue (line), pmatch, guest, lcb->guest_req))
				{
					Assert (lcb->guest_req->lang != LANG_AUTO);
					if (isGuestRequestConsistent(lcb->guest_req))
						guestRequestSubmit (lcb->guest_req);
					guestRequestClear (lcb->guest_req);
				}
			}
		}
		else if (patbuf->type == PTRN_CALLBACK)
			result = matchCallbackPattern (line, patbuf, pmatch);
		else
		{
			Assert ("invalid pattern type" == NULL);
			result = false;
		}
	}
	else
		entry->statistics.unmatch++;
	return result;
}

static bool matchMultilineRegexPattern (struct lregexControlBlock *lcb,
										const vString* const allLines,
										regexTableEntry *entry)
{
	const char *start;
	const char *current;
	off_t offset = 0;
	regexPattern* patbuf = entry->pattern;
	struct mGroupSpec *mgroup = &patbuf->mgroup;
	struct guestSpec  *guest = &patbuf->guest;

	bool result = false;
	regmatch_t pmatch [BACK_REFERENCE_COUNT];
	int match = 0;
	unsigned int delta = 1;

	Assert (patbuf);

	if (patbuf->disabled && *(patbuf->disabled))
		return false;

	current = start = vStringValue (allLines);
	do
	{
		match = regexec (patbuf->pattern, current,
						 BACK_REFERENCE_COUNT, pmatch, 0);
		if (match != 0)
		{
			entry->statistics.unmatch++;
			break;
		}

		if (hasMessage(patbuf))
			printMessage(lcb->owner, patbuf, (current + pmatch[0].rm_so) - start, current, pmatch);

		offset = (current + pmatch [mgroup->forLineNumberDetermination].rm_so)
				 - start;

		entry->statistics.match++;
		if (patbuf->type == PTRN_TAG)
		{
			matchTagPattern (lcb, current, patbuf, pmatch, offset);
			result = true;
		}
		else if (patbuf->type == PTRN_CALLBACK)
			;	/* Not implemented yet */
		else
		{
			Assert ("invalid pattern type" == NULL);
			result = false;
			break;
		}

		if (fillGuestRequest (start, current, pmatch, guest, lcb->guest_req))
		{
			Assert (lcb->guest_req->lang != LANG_AUTO);
			if (isGuestRequestConsistent(lcb->guest_req))
				guestRequestSubmit (lcb->guest_req);
			guestRequestClear (lcb->guest_req);
		}

		delta = (mgroup->nextFromStart
				 ? pmatch [mgroup->forNextScanning].rm_so
				 : pmatch [mgroup->forNextScanning].rm_eo);
		if (delta == 0)
		{
			unsigned int pos = current - start;
			error (WARNING,
				   "a multi line regex pattern doesn't advance the input cursor: %s",
				   patbuf->pattern_string);
			error (WARNING, "Language: %s, input file: %s, pos: %u",
				   getLanguageName (lcb->owner), getInputFileName(), pos);
			break;
		}
		current += delta;

	} while (current < start + vStringLength (allLines));

	return result;
}

/* PUBLIC INTERFACE */

/* Match against all patterns for specified language. Returns true if at least
 * on pattern matched.
 */
extern bool matchRegex (struct lregexControlBlock *lcb, const vString* const line)
{
	bool result = false;
	unsigned int i;
	for (i = 0  ;  i < ptrArrayCount(lcb->entries[REG_PARSER_SINGLE_LINE])  ;  ++i)
	{
		regexTableEntry *entry = ptrArrayItem(lcb->entries[REG_PARSER_SINGLE_LINE], i);
		regexPattern *ptrn = entry->pattern;

		Assert (ptrn);

		if ((ptrn->xtagType != XTAG_UNKNOWN)
			&& (!isXtagEnabled (ptrn->xtagType)))
				continue;

		if (matchRegexPattern (lcb, line, entry))
		{
			result = true;
			if (ptrn->exclusive)
				break;
		}
	}
	return result;
}

extern void notifyRegexInputStart (struct lregexControlBlock *lcb)
{
	lcb->currentScope = CORK_NIL;

	ptrArrayClear (lcb->tstack);
	guestRequestClear (lcb->guest_req);
}

extern void notifyRegexInputEnd (struct lregexControlBlock *lcb)
{
	unsigned long endline = getInputLineNumber ();
	fillEndLineFieldOfUpperScopes (lcb, endline);
}

extern void findRegexTagsMainloop (int (* driver)(void))
{
	/* merely read all lines of the file */
	while (driver () != EOF)
		;
}

static int fileReadLineDriver(void)
{
	return (readLineFromInputFile () == NULL)? EOF: 1;
}

extern void findRegexTags (void)
{
	findRegexTagsMainloop (fileReadLineDriver);
}

static bool hasScopeActionInRegex0(ptrArray *entries)
{
	for (unsigned int i = 0; i < ptrArrayCount(entries); i++)
	{
		regexTableEntry *entry = ptrArrayItem(entries, i);
		Assert (entry && entry->pattern);
		if (entry->pattern->scopeActions)
			return true;
	}
	return false;
}

extern bool hasScopeActionInRegex (struct lregexControlBlock *lcb)
{
	ptrArray *entries;

	entries = lcb->entries[REG_PARSER_SINGLE_LINE];
	if (hasScopeActionInRegex0 (entries))
		return true;

	for (unsigned int i = 0; i < ptrArrayCount(lcb->tables); i++)
	{
		struct regexTable *table = ptrArrayItem(lcb->tables, i);
		if (hasScopeActionInRegex0 (table->entries))
			return true;
	}

	return false;
}

static char *escapeRegexPattern (const char* pattern)
{
	vString *p = vStringNew ();

	while (*pattern != '\0')
	{
		char c = *pattern;
		if (c == '\n')
			vStringCatS(p, "\\n");
		else if (c == '\t')
			vStringCatS(p, "\\t");
		else if (c == '\\')
			vStringCatS(p, "\\\\");
		else
			vStringPut(p, c);

		pattern++;
	}

	return vStringDeleteUnwrap (p);
}

static regexPattern *addTagRegexInternal (struct lregexControlBlock *lcb,
										  int table_index,
					  enum regexParserType regptype,
					  const char* const regex,
					  const char* const name,
					  const char* const kinds,
					  const char* const flags,
					  bool *disabled)
{
	Assert (regex != NULL);
	Assert (name != NULL);

	if (!regexAvailable)
		return NULL;

	regex_t* const cp = compileRegex (regptype, regex, flags);
	if (cp == NULL)
		return NULL;

	char kindLetter;
	char* kindName;
	char* description;
	kindDefinition* fileKind;

	bool explictly_defined =  parseKinds (kinds, &kindLetter, &kindName, &description);
	fileKind = getLanguageKind (lcb->owner, KIND_FILE_INDEX);
	if (kindLetter == fileKind->letter)
		error (FATAL,
			   "Kind letter \'%c\' used in regex definition \"%s\" of %s language is reserved in ctags main",
			   kindLetter,
			   regex,
			   getLanguageName (lcb->owner));
	else if (!isalpha ((unsigned char)kindLetter))
		error (FATAL,
			   "Kind letter must be an alphabetical character: \"%c\"",
			   kindLetter);

	if (strcmp (kindName, fileKind->name) == 0)
		error (FATAL,
			   "Kind name \"%s\" used in regex definition \"%s\" of %s language is reserved in ctags main",
			   kindName,
			   regex,
			   getLanguageName (lcb->owner));

	const char *option_bsae = (regptype == REG_PARSER_SINGLE_LINE? "regex"        :
							   regptype == REG_PARSER_MULTI_LINE ? "mline-regex"  :
							   regptype == REG_PARSER_MULTI_TABLE? "_mtable-regex":
							   NULL);
	Assert (option_bsae);

	for (const char * p = kindName; *p; p++)
	{
		if (p == kindName)
		{
			if (!isalpha(*p))
				error (FATAL,
					   "A kind name doesn't start with an alphabetical character: "
					   "'%s' in \"--%s-%s\" option",
					   kindName,
					   option_bsae,
					   getLanguageName (lcb->owner));
		}
		else
		{
			/*
			 * People may object to this error.
			 * Searching github repositories, I found not a few .ctags files
			 * in which Exuberant-ctags users define kind names with whitespaces.
			 * "FATAL" error breaks the compatibility.
			 */
			if (!isalnum(*p))
				error (/* regptype == REG_PARSER_SINGLE_LINE? WARNING: */ FATAL,
					   "Non-alphanumeric char is used in kind name: "
					   "'%s' in \"--%s-%s\" option",
					   kindName,
					   option_bsae,
					   getLanguageName (lcb->owner));

		}
	}

	regexPattern *rptr = addCompiledTagPattern (lcb, table_index,
												regptype, cp, name,
												kindLetter, kindName, description, flags,
												explictly_defined,
												disabled);
	rptr->pattern_string = escapeRegexPattern(regex);

	eFree (kindName);
	if (description)
		eFree (description);

	if (*name == '\0')
	{
		if (rptr->exclusive || rptr->scopeActions & SCOPE_PLACEHOLDER
			|| rptr->anonymous_tag_prefix
			|| regptype == REG_PARSER_MULTI_TABLE
			|| rptr->guest.lang.type != GUEST_LANG_UNKNOWN
			)
			rptr->accept_empty_name = true;
		else
			error (WARNING, "%s: regexp missing name pattern", regex);
	}

	return rptr;
}

extern void addTagRegex (struct lregexControlBlock *lcb,
			 const char* const regex,
			 const char* const name,
			 const char* const kinds,
			 const char* const flags,
			 bool *disabled)
{
	addTagRegexInternal (lcb, TABLE_INDEX_UNUSED,
						 REG_PARSER_SINGLE_LINE, regex, name, kinds, flags, disabled);
}

extern void addTagMultiLineRegex (struct lregexControlBlock *lcb, const char* const regex,
								  const char* const name, const char* const kinds, const char* const flags,
								  bool *disabled)
{
	addTagRegexInternal (lcb, TABLE_INDEX_UNUSED,
						 REG_PARSER_MULTI_LINE, regex, name, kinds, flags, disabled);
}

extern void addTagMultiTableRegex(struct lregexControlBlock *lcb,
								  const char* const table_name,
								  const char* const regex,
								  const char* const name, const char* const kinds, const char* const flags,
								  bool *disabled)
{
	int table_index = getTableIndexForName (lcb, table_name);

	if (table_index < 0)
		error (FATAL, "unknown table name: %s", table_name);

	addTagRegexInternal (lcb, table_index, REG_PARSER_MULTI_TABLE, regex, name, kinds, flags,
						 disabled);
}

extern void addCallbackRegex (struct lregexControlBlock *lcb,
			      const char* const regex,
			      const char* const flags,
			      const regexCallback callback,
			      bool *disabled,
			      void * userData)
{
	Assert (regex != NULL);

	if (!regexAvailable)
		return;


	regex_t* const cp = compileRegex (REG_PARSER_SINGLE_LINE, regex, flags);
	if (cp != NULL)
	{
		regexPattern *rptr = addCompiledCallbackPattern (lcb, cp, callback, flags,
														 disabled, userData);
		rptr->pattern_string = escapeRegexPattern(regex);
	}
}

static void addTagRegexOption (struct lregexControlBlock *lcb,
							   enum regexParserType regptype,
							   const char* const pattern)
{
	if (!regexAvailable)
		return;

	int table_index = TABLE_INDEX_UNUSED;
	char * regex_pat = NULL;
	char *name, *kinds, *flags;


	if (regptype == REG_PARSER_MULTI_TABLE)
	{
		const char *c;
		for (c = pattern; *c; c++)
		{
			if (! (isalnum(*c) || *c == '_'))
			{
				if (*c &&  (*(c + 1) != '^'))
				{
					vString *tmp = vStringNew ();

					/* Put '^' as prefix for the pattern */
					vStringPut(tmp, *c);
					vStringPut(tmp, '^');
					vStringCatS(tmp, c + 1);
					regex_pat = vStringDeleteUnwrap(tmp);
				}
				else
					regex_pat = eStrdup (c);
				break;
			}
		}

		if (regex_pat == NULL || *regex_pat == '\0')
			error (FATAL, "wrong mtable pattern specification: %s", pattern);

		char *table_name = eStrndup(pattern, c - pattern);
		table_index = getTableIndexForName (lcb, table_name);
		if (table_index < 0)
			error (FATAL, "unknown table name: %s (in %s)", table_name, pattern);
		eFree(table_name);
	}
	else
		regex_pat = eStrdup (pattern);

	if (parseTagRegex (regptype, regex_pat, &name, &kinds, &flags))
		addTagRegexInternal (lcb, table_index, regptype, regex_pat, name, kinds, flags,
							 NULL);

	eFree (regex_pat);
}

extern void processTagRegexOption (struct lregexControlBlock *lcb,
								   enum regexParserType regptype,
								   const char* const parameter)
{
	if (parameter == NULL  ||  parameter [0] == '\0')
		clearPatternSet (lcb);
	else if (parameter [0] != '@')
		addTagRegexOption (lcb, regptype, parameter);
	else if (! doesFileExist (parameter + 1))
		error (WARNING, "cannot open regex file");
	else
	{
		const char* regexfile = parameter + 1;

		verbose ("open a regex file: %s\n", regexfile);
		MIO* const mio = mio_new_file (regexfile, "r");
		if (mio == NULL)
			error (WARNING | PERROR, "%s", regexfile);
		else
		{
			vString* const regex = vStringNew ();
			while (readLineRaw (regex, mio))
			{
				if (vStringLength (regex) > 1 && vStringValue (regex)[0] != '\n')
					addTagRegexOption (lcb, regptype, vStringValue (regex));
			}
			mio_unref (mio);
			vStringDelete (regex);
		}
	}
}

/*
*   Regex option parsing
*/

extern void printRegexFlags (bool withListHeader, bool machinable, FILE *fp)
{
	struct colprintTable * table;

	table = flagsColprintTableNew ();

	flagsColprintAddDefinitions (table, regexFlagDefs,  ARRAY_SIZE (regexFlagDefs));
	flagsColprintAddDefinitions (table, prePtrnFlagDef, ARRAY_SIZE (prePtrnFlagDef));
	flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
	flagsColprintAddDefinitions (table, scopePtrnFlagDef, ARRAY_SIZE (scopePtrnFlagDef));
	flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));

	flagsColprintTablePrint (table, withListHeader, machinable, fp);
	colprintTableDelete(table);
}

extern void printMultilineRegexFlags (bool withListHeader, bool machinable, FILE *fp)
{
	struct colprintTable * table;

	table = flagsColprintTableNew ();

	flagsColprintAddDefinitions (table, regexFlagDefs,  ARRAY_SIZE (regexFlagDefs));
	flagsColprintAddDefinitions (table, multilinePtrnFlagDef, ARRAY_SIZE (multilinePtrnFlagDef));
	flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
	flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));

	flagsColprintTablePrint (table, withListHeader, machinable, fp);
	colprintTableDelete(table);
}

extern void printMultitableRegexFlags (bool withListHeader, bool machinable, FILE *fp)
{
	struct colprintTable * table;

	table = flagsColprintTableNew ();

	flagsColprintAddDefinitions (table, regexFlagDefs,  ARRAY_SIZE (regexFlagDefs));
	flagsColprintAddDefinitions (table, multilinePtrnFlagDef, ARRAY_SIZE (multilinePtrnFlagDef));
	flagsColprintAddDefinitions (table, multitablePtrnFlagDef, ARRAY_SIZE (multitablePtrnFlagDef));
	flagsColprintAddDefinitions (table, guestPtrnFlagDef, ARRAY_SIZE (guestPtrnFlagDef));
	flagsColprintAddDefinitions (table, scopePtrnFlagDef, ARRAY_SIZE (scopePtrnFlagDef));
	flagsColprintAddDefinitions (table, commonSpecFlagDef, ARRAY_SIZE (commonSpecFlagDef));

	flagsColprintTablePrint (table, withListHeader, machinable, fp);
	colprintTableDelete(table);
}

extern void freeRegexResources (void)
{
	/* TODO: SHOULD BE REMOVED */
}

extern bool regexNeedsMultilineBuffer (struct lregexControlBlock *lcb)
{
	if  (ptrArrayCount(lcb->entries [REG_PARSER_MULTI_LINE]) > 0)
		return true;
	else if (ptrArrayCount(lcb->tables) > 0)
		return true;
	else
		return false;
}

extern bool matchMultilineRegex (struct lregexControlBlock *lcb, const vString* const allLines)
{
	bool result = false;

	unsigned int i;

	for (i = 0; i < ptrArrayCount(lcb->entries [REG_PARSER_MULTI_LINE]); ++i)
	{
		regexTableEntry *entry = ptrArrayItem(lcb->entries [REG_PARSER_MULTI_LINE], i);
		Assert (entry && entry->pattern);

		if ((entry->pattern->xtagType != XTAG_UNKNOWN)
			&& (!isXtagEnabled (entry->pattern->xtagType)))
			continue;

		result = matchMultilineRegexPattern (lcb, allLines, entry) || result;
	}
	return result;
}

static int getTableIndexForName (const struct lregexControlBlock *const lcb, const char *name)
{
	unsigned int i;

	for (i = 0; i < ptrArrayCount(lcb->tables); i++)
	{
		struct regexTable *table = ptrArrayItem(lcb->tables, i);
		if (strcmp (table->name, name) == 0)
			return (int)i;
	}

	return TABLE_INDEX_UNUSED;
}

extern void addRegexTable (struct lregexControlBlock *lcb, const char *name)
{
	const char *c;
	for (c = name; *c; c++)
		if (! (isalnum(*c) || *c == '_'))
			error (FATAL, "`%c' in \"%s\" is not acceptable as part of table name", *c, name);

	if (getTableIndexForName(lcb, name) >= 0)
	{
		error (WARNING, "regex table \"%s\" is already defined", name);
		return;
	}

	struct regexTable *table = xCalloc(1, struct regexTable);
	table->name = eStrdup (name);
	table->entries = ptrArrayNew(deleteTableEntry);

	ptrArrayAdd (lcb->tables, table);
}

static void dumpSstack(FILE* fp, int scope)
{
	tagEntryInfo *entry;
	fprintf (fp, "scope : ");
	while ((entry = getEntryInCorkQueue (scope)))
	{
		fprintf(fp, "%s", entry->name);

		scope = entry->extensionFields.scopeIndex;
		if (scope != CORK_NIL)
			fprintf(fp, "%c", '/');
	}
	fprintf (fp, "\n");
}

static void dumpTstack(FILE* fp, ptrArray *tstack)
{
	for (unsigned int i = ptrArrayCount(tstack); i > 0; i--)
	{
		char tmp[2];
		struct regexTable *t = ptrArrayItem(tstack, i - 1);
		if (i == 1)
			tmp[0] = '\0';
		else
		{
			tmp[0] = '/';
			tmp[1] = '\0';
		}
		fprintf(fp, "%s%s", t->name, tmp);
	}
	fprintf(fp, "\n");
}

static void printInputLine(FILE* vfp, const char *c, const off_t offset)
{
	vString *v = vStringNew ();

	for (; *c && (*c != '\n'); c++)
		vStringPut(v, *c);

	if (vStringLength (v) == 0 && *c == '\n')
		vStringCatS (v, "\\n");

	fprintf (vfp, "\ninput : \"%s\" L%lu\n",
			 vStringValue (v),
			 getInputLineNumberForFileOffset(offset));
	vStringDelete(v);
}

static void printMultitableMessage(const langType language,
								   const char *const tableName,
								   const unsigned int index,
								   const regexPattern *const ptrn,
								   const off_t offset,
								   const char *const current,
								   const regmatch_t* const pmatch)
{
	vString *msg;

	Assert (ptrn);
	Assert (ptrn->message.selection > 0);
	Assert (ptrn->message.message_string);

	msg = substitute (current, ptrn->message.message_string, BACK_REFERENCE_COUNT, pmatch);

	error (ptrn->message.selection, "%sMessage from mtable<%s/%s[%2u]>: %s (%s:%lu)",
		   (ptrn->message.selection == FATAL ? "Fatal: " : ""),
		   getLanguageName (language),
		   tableName,
		   index,
		   vStringValue (msg),
		   getInputFileName (),
		   getInputLineNumberForFileOffset (offset));

	vStringDelete (msg);
}

static struct regexTable * matchMultitableRegexTable (struct lregexControlBlock *lcb,
													  struct regexTable *table, const vString *const start, unsigned int *offset)
{
	struct regexTable *next = NULL;
	const char *current;
	regmatch_t pmatch [BACK_REFERENCE_COUNT];
	const char *cstart = vStringValue(start);
	unsigned int delta;


 restart:
	current = cstart + *offset;

	/* Accept the case *offset == vStringLength(start)
	   because we want an empty regex // still matches empty input. */
	if (*offset > vStringLength(start))
	{
		*offset = vStringLength(start);
		goto out;
	}

	BEGIN_VERBOSE(vfp);
	{
		printInputLine(vfp, current, *offset);
	}
	END_VERBOSE();

	for (unsigned int i = 0; i < ptrArrayCount(table->entries); i++)
	{
		regexTableEntry *entry = ptrArrayItem(table->entries, i);
		regexPattern *ptrn = entry->pattern;
		struct guestSpec  *guest = &ptrn->guest;

		Assert (ptrn);

		BEGIN_VERBOSE(vfp);
		{
			char s[3];
			if (*current == '\n')
			{
				s [0] = '\\';
				s [1] = 'n';
				s [2] = '\0';
			}
			else if (*current == '\t')
			{
				s [0] = '\\';
				s [1] = 't';
				s [2] = '\0';
			}
			else if (*current == '\\')
			{
				s [0] = '\\';
				s [1] = '\\';
				s [2] = '\0';
			}
			else
			{
				s[0] = *current;
				s[1] = '\0';
			}

			if (s[1] == '\0')
				fprintf (vfp, "match : '%s' %15s[%2u] /", s, table->name, i);
			else if (s[0] == '\0')
				fprintf (vfp, "match :  '' %15s[%2u] /", table->name, i);
			else
				fprintf (vfp, "match :'%s' %15s[%2u] / ", s, table->name, i);
			fprintf (vfp, "%s/\n", ptrn->pattern_string);
		}
		END_VERBOSE();

		int match = 0;

		if (ptrn->disabled && *(ptrn->disabled))
			continue;

		match = regexec (ptrn->pattern, current,
						 BACK_REFERENCE_COUNT, pmatch, 0);

		if (match == 0)
		{
			entry->statistics.match++;

			if (ptrn->type == PTRN_TAG)
			{
				struct mTableActionSpec *taction = &(ptrn->taction);

				matchTagPattern (lcb, current, ptrn, pmatch,
								 (current
								  + pmatch [ptrn->mgroup.forLineNumberDetermination].rm_so)
								 - cstart);
				BEGIN_VERBOSE(vfp);
				{
					fprintf(vfp, "result: matched %d bytes\n", (int)(pmatch[0].rm_eo));
					dumpSstack (vfp, lcb->currentScope);
				}
				END_VERBOSE();

				if (hasMessage(ptrn))
					printMultitableMessage (lcb->owner, table->name, i, ptrn,
											*offset, current, pmatch);

				if (fillGuestRequest (cstart, current, pmatch, guest, lcb->guest_req))
				{
					Assert (lcb->guest_req->lang != LANG_AUTO);
					if (isGuestRequestConsistent(lcb->guest_req))
						guestRequestSubmit (lcb->guest_req);
					guestRequestClear (lcb->guest_req);
				}

				delta = (ptrn->mgroup.nextFromStart
						 ? pmatch [ptrn->mgroup.forNextScanning].rm_so
						 : pmatch [ptrn->mgroup.forNextScanning].rm_eo);
				*offset += delta;

				switch (taction->action)
				{
				case TACTION_NOP:
					BEGIN_VERBOSE(vfp);
					{
						fprintf(vfp, "action: NOP in {%s}, stack: /", table->name);
						dumpTstack(vfp, lcb->tstack);
					}
					END_VERBOSE();
					break;
				case TACTION_ENTER:
					/* TODO: Limit the depth of tstack.  */
					ptrArrayAdd (lcb->tstack,
								 taction->continuation_table
								 ? taction->continuation_table
								 : table);
					next = taction->table;
					BEGIN_VERBOSE(vfp);
					{
						if (taction->continuation_table)
							fprintf(vfp, "action: [enter] to {%s}, cont: {%s}, stack: /",
									next->name,
									taction->continuation_table->name);
						else
							fprintf(vfp, "action: [enter] to {%s}, stack: /", next->name);
						dumpTstack(vfp, lcb->tstack);
					}
					END_VERBOSE();
					break;
				case TACTION_LEAVE:
					BEGIN_VERBOSE(vfp);
					{
						fprintf(vfp, "action: [leave] from {%s}, stack: /", table->name);
						dumpTstack(vfp, lcb->tstack);
					}
					END_VERBOSE();
					if (ptrArrayCount (lcb->tstack) == 0)
					{
						error (WARNING, "leave is specified as regex table action but the table stack is empty");
						return NULL;
					}
					next = ptrArrayLast(lcb->tstack);
					ptrArrayRemoveLast (lcb->tstack);
					break;
				case TACTION_JUMP:
					next = taction->table;
					BEGIN_VERBOSE(vfp);
					{
						fprintf(vfp, "action: [jump] from {%s} to {%s}, stack: /", table->name, next->name);
						dumpTstack(vfp, lcb->tstack);
					}
					END_VERBOSE();

					break;
				case TACTION_RESET:
					next = taction->table;
					BEGIN_VERBOSE(vfp);
					{
						fprintf(vfp, "action: [reset] to {%s}, stack: /", next->name);
					}
					END_VERBOSE();

					ptrArrayClear (lcb->tstack);
					break;
				case TACTION_QUIT:
					BEGIN_VERBOSE(vfp);
					{
						fprintf(vfp, "action: [quit], stack: /");
						dumpTstack(vfp, lcb->tstack);
					}
					END_VERBOSE();
					return NULL;
				}

				if (next)
					break;

				if (delta == 0)
				{
					error (WARNING, "Forcefully advance the input pos because");
					error (WARNING, "following conditions for entering infinite loop are satisfied:");
					error (WARNING, "+ matching the pattern succeeds,");
					error (WARNING, "+ the next table is not given, and");
					error (WARNING, "+ the input file pos doesn't advance.");
					error (WARNING, "Language: %s, input file: %s, pos: %u",
						   getLanguageName (lcb->owner), getInputFileName(), *offset);
					++*offset;
				}
			}
			else if (ptrn->type == PTRN_CALLBACK)
				;	/* Not implemented yet */
			else
			{
				Assert ("invalid pattern type" == NULL);
				break;
			}
			goto restart;
		}
		else
			entry->statistics.unmatch++;
	}
 out:
	if (next == NULL && ptrArrayCount (lcb->tstack) > 0)
	{
		static int apop_count = 0;
		next = ptrArrayLast(lcb->tstack);
		verbose("result: no match - autopop<%d> from {%s} to {%s} @ %lu\n", apop_count++, table->name, next->name,
				getInputLineNumberForFileOffset(*offset));
		ptrArrayRemoveLast (lcb->tstack);
	}
	return next;
}

extern void extendRegexTable (struct lregexControlBlock *lcb, const char *src, const char *dist)
{

	int i;
	struct regexTable * src_table;
	struct regexTable * dist_table;

	verbose ("extend regex table  \"%s\" with \"%s\"\n", dist, src);

	i = getTableIndexForName (lcb, src);
	if (i < 0)
		error (FATAL, "no such regex table in %s: %s", getLanguageName(lcb->owner), src);
	src_table = ptrArrayItem(lcb->tables, i);

	i = getTableIndexForName (lcb, dist);
	if (i < 0)
		error (FATAL, "no such regex table in %s: %s", getLanguageName(lcb->owner), dist);
	dist_table = ptrArrayItem(lcb->tables, i);

	for (i = 0; i < (int)ptrArrayCount(src_table->entries); i++)
	{
		regexTableEntry *entry = ptrArrayItem (src_table->entries, i);
		ptrArrayAdd(dist_table->entries, newRefPatternEntry(entry));
	}
}

extern void printMultitableStatistics (struct lregexControlBlock *lcb)
{
	if (ptrArrayCount(lcb->tables) == 0)
		return;

	fprintf(stderr, "\nMTABLE REGEX STATISTICS of %s\n", getLanguageName (lcb->owner));
	fputs("==============================================\n", stderr);
	for (unsigned int i = 0; i < ptrArrayCount(lcb->tables); i++)
	{
		struct regexTable *table = ptrArrayItem (lcb->tables, i);
		fprintf(stderr, "%s\n", table->name);
		fputs("-----------------------\n", stderr);
		for (unsigned int j = 0; j < ptrArrayCount(table->entries); j++)
		{
			regexTableEntry *entry = ptrArrayItem (table->entries, j);
			Assert (entry && entry->pattern);
			fprintf(stderr, "%10u/%-10u%-40s ref: %d\n",
					entry->statistics.match,
					entry->statistics.unmatch + entry->statistics.match,
					entry->pattern->pattern_string,
					entry->pattern->refcount);
		}
		fputc('\n', stderr);
	}
}

extern bool matchMultitableRegex (struct lregexControlBlock *lcb, const vString* const allLines)
{
	if (ptrArrayCount (lcb->tables) == 0)
		return false;

	struct regexTable *table = ptrArrayItem (lcb->tables, 0);
	unsigned int offset = 0;

	int motionless_counter = 0;
	unsigned int last_offset;


	while (table)
	{
		last_offset = offset;
		table = matchMultitableRegexTable(lcb, table, allLines, &offset);

		if (last_offset == offset)
			motionless_counter++;
		else
			motionless_counter = 0;

		if (motionless_counter > MTABLE_MOTIONLESS_MAX)
		{
			error (WARNING, "mtable<%s/%s>: the input cursor stays at %u in %s so long though the tables are switched",
				   getLanguageName (lcb->owner),
				   table->name, offset, getInputFileName ());
			break;
		}

		if (table && (ptrArrayCount (lcb->tstack) > MTABLE_STACK_MAX_DEPTH))
		{
			unsigned int i;
			struct regexTable *t;

			error (WARNING, "mtable<%s/%s>: the tenter/tleave stack overflows at %u in %s",
				   getLanguageName (lcb->owner),
				   table->name, offset, getInputFileName ());
			error (WARNING, "DUMP FROM THE TOP:");
			/* TODO: use dumpTstack */
			for (i = ptrArrayCount(lcb->tstack); 0 < i; --i)
			{
				t = ptrArrayItem (lcb->tstack, i - 1);
				error (WARNING, "%3u %s", i - 1, t->name);
			}

			break;
		}
	}

	return true;
}

static int  makePromiseForAreaSpecifiedWithOffsets (const char *parser,
													off_t startOffset,
													off_t endOffset)
{
	unsigned long startLine = getInputLineNumberForFileOffset(startOffset);
	unsigned long endLine = getInputLineNumberForFileOffset(endOffset);
	unsigned long startLineOffset = getInputFileOffsetForLine (startLine);
	unsigned long endLineOffset = getInputFileOffsetForLine (endLine);

	return makePromise (parser,
						startLine, startOffset - startLineOffset,
						endLine, endOffset - endLineOffset,
						startOffset - startLineOffset);
}

static struct guestRequest *guestRequestNew (void)
{
	struct guestRequest *r = xMalloc (1, struct guestRequest);


	guestRequestClear (r);
	return r;
}

static void   guestRequestDelete (struct guestRequest *r)
{
	eFree (r);
}

static bool   guestRequestIsFilled(struct guestRequest *r)
{
	return (r->lang_set && (r->boundary + 0)->offset_set && (r->boundary + 1)->offset_set);
}

static void   guestRequestClear (struct guestRequest *r)
{
	r->lang_set = false;
	r->boundary[BOUNDARY_START].offset_set = false;
	r->boundary[BOUNDARY_END].offset_set = false;
}

static void   guestRequestSubmit (struct guestRequest *r)
{
	const char *langName = getLanguageName (r->lang);
	verbose ("guestRequestSubmit: %s; "
			 "range: %"PRId64" - %"PRId64"\n",
			 langName,
			 (int64_t)r->boundary[BOUNDARY_START].offset,
			 (int64_t)r->boundary[BOUNDARY_END].offset);
	makePromiseForAreaSpecifiedWithOffsets (langName,
											r->boundary[BOUNDARY_START].offset,
											r->boundary[BOUNDARY_END].offset);
}

/* Return true if available. */
extern bool checkRegex (void)
{
#if defined (CHECK_REGCOMP)
	{
		/* Check for broken regcomp() on Cygwin */
		regex_t patbuf;
		int errcode;
		if (regcomp (&patbuf, "/hello/", 0) != 0)
			error (WARNING, "Disabling broken regex");
		else
			regexAvailable = true;
	}
#else
	/* We are using bundled regex engine. */
	regexAvailable = true;
#endif
	return regexAvailable;
}
