/*
**  OSSP lmtp2nntp - Mail to News Gateway
**  Copyright (c) 2001-2003 Ralf S. Engelschall <rse@engelschall.com>
**  Copyright (c) 2001-2003 The OSSP Project <http://www.ossp.org/>
**  Copyright (c) 2001-2003 Cable & Wireless Germany <http://www.cw.com/de/>
**
**  This file is part of OSSP lmtp2nntp, an LMTP speaking local
**  mailer which forwards mails as Usenet news articles via NNTP.
**  It can be found at http://www.ossp.org/pkg/tool/lmtp2nntp/.
**
**  This program is free software; you can redistribute it and/or
**  modify it under the terms of the GNU General Public  License
**  as published by the Free Software Foundation; either version
**  2.0 of the License, or (at your option) any later version.
**
**  This program is distributed in the hope that it will be useful,
**  but WITHOUT ANY WARRANTY; without even the implied warranty of
**  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
**  General Public License for more details.
**
**  You should have received a copy of the GNU General Public License
**  along with this file; if not, write to the Free Software
**  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
**  USA, or contact the OSSP project <ossp@ossp.org>.
**
**  lmtp2nntp_msg.c: mail message manipulation library
*/

#include <stdlib.h>
#include <stdio.h>

#include "lmtp2nntp_msg.h"
#include "lmtp2nntp_argz.h"
#include "lmtp2nntp_common.h" /* only required for logbook() */
#include "tai.h"
#include "str.h"

/* third party */
#include "l2.h"

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#if defined(HAVE_DMALLOC_H) && defined(WITH_DMALLOC)
#include "dmalloc.h"
#endif

msg_t *msg_create(val_t *prival)
{
    ex_t ex;
    msg_t *msg;

    if (prival == NULL)
        return NULL;

    if ((msg = (msg_t *)malloc(sizeof(msg_t))) == NULL)
        return NULL;

    msg->azEnvgroups = NULL;
    msg->asEnvgroups = 0;
    msg->cpMsg = NULL;
    msg->azHeaders = NULL;
    msg->asHeaders = 0;
    msg->hdFirst = NULL;
    msg->cpFid = NULL;
    msg->cpBody = NULL;
    msg->cpMsgid = NULL;
    msg->mail_from = NULL;
    msg->azRcpt = NULL;
    msg->asRcpt = 0;
    msg->azNewsgroups = NULL;
    msg->asNewsgroups = 0;
    msg->l2 = NULL; /* this is a copy only */
    msg->prival = prival;

    /* create printable variables context and mount it into the private application context */
    try {
        val_reg(msg->prival, "msg", VAL_TYPE_PTR, "msg context structure", NULL);
        val_set(msg->prival, "msg", msg);
        }
    catch (ex)
        rethrow;

    return msg;
}

void msg_destroy(msg_t *msg)
{
    if (msg == NULL)
        return;

    if (msg->azEnvgroups != NULL)
        free(msg->azEnvgroups);
    if (msg->cpMsg != NULL)
        free(msg->cpMsg);
    if (msg->azHeaders != NULL)
        free(msg->azHeaders);
    if (msg->cpFid != NULL)
        free(msg->cpFid);
    if (msg->cpBody != NULL)
        free(msg->cpBody);
    if (msg->cpMsgid != NULL)
        free(msg->cpMsgid);
    if (msg->mail_from != NULL)
        free(msg->mail_from);
    if (msg->azRcpt != NULL)
        free(msg->azRcpt);
    if (msg->azNewsgroups != NULL)
        free(msg->azNewsgroups);
    msg->l2 = NULL; /* this is a copy only, the "parent" needs to clean this up */
    if (msg->prival != NULL)
        val_unreg(msg->prival, "msg");

    free(msg);
    return;
}

msg_rc_t msg_split(msg_t *msg)
{
    char        *cpName;
    char        *cpValue;
    char        *cpRem;     /* Remainder */
    char        *cp;
    char        *cpHeaders;

    /* INPUTS
     *
     * msg->cpMsg
     * must contain the wholly RFC0822 formatted message with native
     * (unescaped) dots at the beginning of a line, the 'From ' envelope,
     * headers, double newline, body, NUL, no trailing dot;
     *
     * OUTPUTS
     *
     * msg->cpMsg
     * free()d and set to NULL
     *
     * msg->azHeaders, msg->asHeaders contains the headers in argz format, one
     * logical NUL-terminated line per header which might be wrapped into
     * multiple '\n'-ended physical lines. The "From " envelope, "Received:",
     * "Path:", "To:" and "Cc:" headers are removed silently. The
     * "Newsgroups:" and "Message-ID" headers are removed and their values are
     * stored in separate structures (see below).
     *
     * msg->cpBody
     * contains the unmodified body of the message, NUL-terminated, no
     * trailing dot.
     *
     * msg->cpMsgid
     * contains the message id including surrounding angle brackets.
     *
     * msg->azNewsgroups, asNewsgroups
     * is a argz-type array of strings containing the Newsgroups based on the
     * header information.
     */

    logbook(msg->l2, L2_LEVEL_DEBUG, "split message into header and body");
    if (str_parse(msg->cpMsg, "m/((?:.*?)\\n)\\n(.*)$/s", &cpHeaders, &msg->cpBody) <= 0)
        return MSG_ERR_SPLITHEADBODY;

    free(msg->cpMsg);
    msg->cpMsg = NULL;

    logbook(msg->l2, L2_LEVEL_DEBUG, "replace envelope-from by -FE: pseudoheader");
    /* This eliminates the special case of having one header, which is really
     * an embedded envelope, not ending with a colon while all others do.
     */
    if (strncasecmp(cpHeaders, "From", 4) == 0)
        memcpy(cpHeaders, "-EF:", 4);

    logbook(msg->l2, L2_LEVEL_DEBUG, "unwrap header lines");
    /* poor man's s///g simulator as current str library doesn't support global substitution */
    while (str_parse(cpHeaders, "s/(.*?)\\n[ \\t]+(.*)/$1 $2/s", &cpRem) > 0) {
        free(cpHeaders);
        cpHeaders = cpRem;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "split header lines into names and values");
    while (str_parse(cpHeaders, "m/^[> \\t]*([\\x21-\\x7e]+?:)[ \\t]*([^\\n]*?)[ \\t]*\\n(.*)/s", &cpName, &cpValue, &cpRem) > 0) {
        free(cpHeaders);
        cpHeaders = cpRem;
        argz_add(&msg->azHeaders, &msg->asHeaders, cpName);
        argz_add(&msg->azHeaders, &msg->asHeaders, cpValue);
        free(cpName);
        free(cpValue);
    }
    free(cpHeaders);

    logbook(msg->l2, L2_LEVEL_DEBUG, "check for headers we care about and do whatever neccessary");
    msg->azNewsgroups = NULL;
    msg->asNewsgroups = 0;
    cp = msg->azHeaders;
    while (cp != NULL) {
        logbook(msg->l2, L2_LEVEL_DEBUG, "processing header \"%s\" in split", cp);
        if (strcasecmp("Received:", cp) == 0) {
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  name  */
            if ((msg->cpFid == NULL) &&
                (str_parse(cp, "m/\\sid\\s+<?([\\w\\d]{1,30})/i", &msg->cpFid) > 0))
                    logbook(msg->l2, L2_LEVEL_DEBUG, "found foreign-ID \"%s\" for logging", msg->cpFid);
            argz_delete(&msg->azHeaders, &msg->asHeaders, cp);             /* del  value */
            continue;
        }
        if (strcasecmp("Newsgroups:", cp) == 0) {
            if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
                break;
            if (argz_add(&msg->azNewsgroups, &msg->asNewsgroups, cp) != 0) /* get  value */
                return MSG_ERR_MEM;
            if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
                break;
            continue;
        }
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
            break;
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next name  */
            break;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "checking Newsgroups");
    if (msg->azNewsgroups != NULL) {
        argz_stringify(msg->azNewsgroups, msg->asNewsgroups, ',');
        if (argz_create_sep(msg->azNewsgroups, ',', &msg->azNewsgroups, &msg->asNewsgroups) != 0)
            return MSG_ERR_MEM;
    }

    logbook(msg->l2, L2_LEVEL_DEBUG, "split complete");
    return MSG_OK;
}

msg_rc_t msg_join(msg_t *msg)
{
    char        *cp;
    char        *cpRem;
    char       **aHeaders;
    int          i;
    int          o;
    char        *cpCut;
    char        *cpWrap;
    char         c;
    char         cOld;
    int          n;
    char        *cpHeaders;
    char        *azNewheaders;
    size_t       asNewheaders;

    logbook(msg->l2, L2_LEVEL_DEBUG, "verify Newsgroups");
    if ((msg->azNewsgroups == NULL) || (msg->asNewsgroups == 0))
        return MSG_ERR_JOINGROUPNONE;
    argz_stringify(msg->azNewsgroups, msg->asNewsgroups, ',');
    if (strlen(msg->azNewsgroups) == 0)
        return MSG_ERR_JOINGROUPEMPTY;
    argz_add(&msg->azHeaders, &msg->asHeaders, "Newsgroups:");
    argz_add(&msg->azHeaders, &msg->asHeaders, msg->azNewsgroups);

    logbook(msg->l2, L2_LEVEL_DEBUG, "verify Message-ID");
    cp = msg->azHeaders;
    while (cp != NULL) {
        logbook(msg->l2, L2_LEVEL_DEBUG, "processing header \"%s\" in join", cp);
        if (strcasecmp("Message-ID:", cp) == 0) {
            if (msg->cpMsgid != NULL)
                return MSG_ERR_JOINIDMULTI;
            if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
                break;
            if ((cp == NULL) || (strlen(cp) == 0))                             /* get  value */
                return MSG_ERR_JOINIDEMPTY;
            if ((msg->cpMsgid = strdup(cp)) == NULL)
                return MSG_ERR_MEM;
            if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)  /* next value */
                break;
            continue;
        }
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)      /* next value */
            break;
        if ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) == NULL)      /* next name  */
            break;
    }
    if (msg->cpMsgid == NULL)
        return MSG_ERR_JOINIDNONE;
    if (strlen(msg->cpMsgid) == 0)
        return MSG_ERR_JOINIDEMPTY;

    logbook(msg->l2, L2_LEVEL_DEBUG, "merge name/value pairs into single string");
    argz_add(&msg->azHeaders, &msg->asHeaders, ""); /* append empty string */
    if ((aHeaders = (char **)malloc((argz_count(msg->azHeaders, msg->asHeaders) + 1) * sizeof(char *))) == NULL)
        return MSG_ERR_MEM;
    argz_extract(msg->azHeaders, msg->asHeaders, aHeaders);
    /* replace the trailing NUL, which is *(cp-1) of the predecessor, with a
     * space at every second string. Break action when terminating NULL string
     * is detected */
    i=0;
    while(1) {
        if ((cp = aHeaders[++i]) == NULL)
            break;
        *(cp-1) = ' ';
        if ((cp = aHeaders[++i]) == NULL)
            break;
    }
    free(aHeaders);

    logbook(msg->l2, L2_LEVEL_DEBUG, "fold headers");
    /* A logical line is split into one or more physical '\n'-terminated
     * lines. The physical line is never longer than WRAPAT characters. This
     * includes the folded data and the header name + colon + space for the
     * first line and WRAPUSING string prefix for all other lines. Leading and
     * trailing blanks of folded lines are removed while blanks inside the
     * line are preserved.  The header is never left alone in a physical line.
     * Fragments exceeding WRAPAT characters without having a blank as a
     * splitting point are forcibly cut at a non-blank character.
     */
    azNewheaders = NULL;
    asNewheaders = 0;
    cp = NULL;
    while ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) != NULL) {
        if (strlen(cp) > WRAPAT) {
            cpRem = cp;
            cpWrap = NULL;
            for (o = 0; (cpRem[o] != ':') && (cpRem[o] != NUL); o++); /* offset name so at least one char of value remains in first line */
            o += 2; /* skip ": " */
            while ((strlen(cpRem) + (cpWrap == NULL ? 0 : strlen(WRAPUSING))) > WRAPAT) {
                for (i = WRAPAT - 1 - (cpWrap == NULL ? 0 :
                    strlen(WRAPUSING)); (i >= o) && !isspace((int)cpRem[i]); i--);
                if (i < o)
                    i = WRAPAT - 1 - (cpWrap == NULL ? 0 : strlen(WRAPUSING) - 1); /* sorry, forced cut at non-blank */
                cpCut = cpRem;
                cpRem += i;
                for (; (isspace((int)*cpRem) && (*cpRem != NUL)); cpRem++); /* skip next lines leading blanks */
                for (; (i >= o) && isspace((int)cpCut[i-1]); i--); /* chop off this lines trailing blanks */
                if (i >= o) { /* only keep line fragment if some non-blanks inside */
                    if (cpWrap == NULL) {
                        if ((cpWrap = (char *)malloc(i+strlen(WRAPUSING)+1)) == NULL)
                            return MSG_ERR_MEM;
                        *cpWrap = NUL;
                        o = 1;
                    }
                    else {
                        if ((cpWrap = (char *)realloc(cpWrap, strlen(cpWrap)+i+strlen(WRAPUSING)+1)) == NULL)
                            return MSG_ERR_MEM;
                        strcat(cpWrap, WRAPUSING);
                    }
                    strncat(cpWrap, cpCut, i);
                }
            }
            if (strlen(cpRem) > 0) {
                if ((cpWrap = (char *)realloc(cpWrap, strlen(cpWrap)+strlen(cpRem)+strlen(WRAPUSING)+1)) == NULL)
                        return MSG_ERR_MEM;
                strcat(cpWrap, WRAPUSING);
                strcat(cpWrap, cpRem);
            }
            argz_add(&azNewheaders, &asNewheaders, cpWrap);
            logbook(msg->l2, L2_LEVEL_DEBUG, "a folded header \"%{text}D\"", cpWrap, strlen(cpWrap));
            free(cpWrap);
        }
        else {
            argz_add(&azNewheaders, &asNewheaders, cp);
            logbook(msg->l2, L2_LEVEL_DEBUG, "verbatim header \"%{text}D\"", cp, strlen(cp));
        }
    }
    free(msg->azHeaders);
    msg->azHeaders = azNewheaders;
    msg->asHeaders = asNewheaders;

    logbook(msg->l2, L2_LEVEL_DEBUG, "strigify headers");
    argz_stringify(msg->azHeaders, msg->asHeaders, '\n');
    cpHeaders = msg->azHeaders;

    /********************************************************************
     * header + CRLF + body + '.' + CRLF + NUL, replacing NL with CRLF *
     ********************************************************************/

    logbook(msg->l2, L2_LEVEL_DEBUG, "assemble header and body");
    n = 0;
    /* count size of headers, reserve space for NL to CRLF conversion */
    for (i = 0; ((c = cpHeaders[i]) != NUL); i++) {
        if (c == '\n')
            n++;
        n++;
    }
    /* if headers don't end with NL, reserve space for CRLF */
    if (i >= 0 && cpHeaders[i - 1] != '\n')
        n+=2;
    /* reserve space for CRLF between headers and body */
    n+=2;
    /* count size of body, reserve space for NL-DOT escape and NL to CRLF conversion */
    cOld = '\n';
    for (i = 0; ((c = msg->cpBody[i]) != NUL); i++) {
        if (c == '\n')
            n++;
        if (c == '.' && cOld == '\n')
            n++;
        n++;
        cOld = c;
    }
    /* if body doesn't end with NL, reserve space for CRLF */
    if (i >= 0 && msg->cpBody[i - 1] != '\n')
        n+=2;
    /* reserve space for terminating '.'-CRLF-NUL at the end of the message */
    n+=4;

    if ((msg->cpMsg = (char *)malloc(n)) == NULL)
        return MSG_ERR_MEM;

    n = 0;
    /* copy headers, do NL to CRLF conversion */
    for (i = 0; ((c = cpHeaders[i]) != NUL); i++) {
        if (c == '\n')
            msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = c;
    }
    /* if headers don't end with NL, append CRLF */
    if (i >= 0 && cpHeaders[i - 1] != '\n') {
        msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = '\n';
    }
    /* add CRLF between headers and body */
    msg->cpMsg[n++] = '\r';
    msg->cpMsg[n++] = '\n';
    /* copy body, do NL-DOT escape and NL to CRLF conversion */
    cOld = '\n';
    for (i = 0; ((c = msg->cpBody[i]) != NUL); i++) {
        if (c == '\n')
            msg->cpMsg[n++] = '\r';
        if (c == '.' && cOld == '\n')
            msg->cpMsg[n++] = '.';
        msg->cpMsg[n++] = c;
        cOld = c;
    }
    /* if body doesn't end with NL, append CRLF */
    if (i >= 0 && msg->cpBody[i - 1] != '\n') {
        msg->cpMsg[n++] = '\r';
        msg->cpMsg[n++] = '\n';
    }
    /* add terminating '.'-CRLF-NUL at the end of the message */
    msg->cpMsg[n++] =  '.';
    msg->cpMsg[n++] = '\r';
    msg->cpMsg[n++] = '\n';
    msg->cpMsg[n]   = NUL;

    logbook(msg->l2, L2_LEVEL_DEBUG, "join complete");
    return MSG_OK;
}

char *msg_error(msg_rc_t rc)
{
    char *str;
                                              str = "MSG: no description";
    if      (rc == MSG_OK                   ) str = "MSG: no error";
    else if (rc == MSG_ERR_MEM              ) str = "MSG: memory";
    else if (rc == MSG_ERR_SPLITHEADBODY    ) str = "MSG: split into header and body failed";
    else if (rc == MSG_ERR_SPLITLEN         ) str = "MSG: header is too short";
    else if (rc == MSG_ERR_SPLITMISSINGFROM ) str = "MSG: header is missing 'From ' envelope";
    else if (rc == MSG_ERR_JOINGROUPNONE    ) str = "MSG: join with no 'Newsgroup'";
    else if (rc == MSG_ERR_JOINGROUPEMPTY   ) str = "MSG: join with empty 'Newsgroup'";
    else if (rc == MSG_ERR_JOINIDNONE       ) str = "MSG: join with no 'Message-ID'";
    else if (rc == MSG_ERR_JOINIDEMPTY      ) str = "MSG: join with empty 'Message-ID'";
    else if (rc == MSG_ERR_JOINIDMULTI      ) str = "MSG: join with multiple 'Message-ID's";
    return str;
}

static void headerdatadestroy(headerdata_t *hdC)
{
    int i;

    if (hdC->ndata > 1) {
        for (i = 0; i < hdC->ndata; i++) {
            if (hdC->data.m[i] == NULL)
                break;
            free(hdC->data.m[i]);
        }
        free (hdC->data.m);
        hdC->ndata = 0;
    } else
    if (hdC->ndata == 1) {
        if (hdC->data.s != NULL)
            free(hdC->data.s);
        hdC->ndata = 0;
    }
}

static void headernamedestroy(headerdata_t *hdC)
{
    if (hdC->name != NULL) {
        free(hdC->name);
        hdC->name = NULL;
    }
}

static void headerdestroy(headerdata_t *hdC)
{
    headerdatadestroy(hdC);
    headernamedestroy(hdC);
    if (hdC->prev != NULL && hdC->prev->next == hdC)
        throw(0,0,0);
    if (hdC->next != NULL && hdC->next->prev == hdC)
        throw(0,0,0);
    free(hdC);
}

#if 0 /* still unused */
static void headerdelete(headerdata_t *hdC)
{
    if (hdC->prev != NULL)
        hdC->prev->next = hdC->next;
    hdC->next = NULL;
    if (hdC->next != NULL)
        hdC->next->prev = hdC->prev;
    hdC->prev = NULL;
    headerdestroy(hdC);
}
#endif

static void headerreplace(headerdata_t *hdC, headerdata_t *hdNew)
{
    hdNew->prev = hdC->prev;
    hdC->prev = NULL;
    hdNew->next = hdC->next;
    hdC->next = NULL;
    if (hdNew->prev != NULL)
        hdNew->prev->next = hdNew;
    if (hdNew->next != NULL)
        hdNew->next->prev = hdNew;
    headerdestroy(hdC);
}

static headerdata_t *headercreate(void)
{
    ex_t ex;
    volatile headerdata_t *hdNew = NULL;
    try {
        hdNew = mallocex(sizeof(headerdata_t));
        hdNew->prev  = NULL;
        hdNew->next  = NULL;
        hdNew->name  = NULL;
        hdNew->ndata = 0;
    }
    catch (ex) {
        if (hdNew != NULL)
            free((headerdata_t *)hdNew);
        rethrow;
    }
    return (headerdata_t *)hdNew;
}

static var_rc_t regex_lookup(
    var_t *var, void *_ctx,
    const char *var_ptr, size_t var_len, int var_idx,
    const char **val_ptr, size_t *val_len, size_t *val_size)
{
    regex_ctx_t *ctx = (regex_ctx_t *)_ctx;
    var_rc_t rc;
    char *cp;
    int i;

    logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d)", var_ptr, var_len);
    rc = VAR_ERR_UNDEFINED_VARIABLE;
    i = atoi(var_ptr); /* works with both '}' and '\0' termination */
    if (i < ctx->nMatch) {
        *val_ptr = ctx->acpMatch[i];
        *val_len = strlen(ctx->acpMatch[i]);
        *val_size = 0;
        rc = VAR_OK;
    }
    if (rc == VAR_OK)
        logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d) ok: result is \"%s\" (%d)", var_ptr, var_len, *val_ptr, *val_len);
    else
        logbook(ctx->l2, L2_LEVEL_DEBUG, "rgx_lookup variable \"%s\" (%d) failed: %s (%d)", var_ptr, var_len, var_strerror(var, rc, &cp) == VAR_OK ? cp : "Unknown Error", rc);
    return rc;
}

static var_rc_t createmessageid(
    val_t *prival,
    const char  *cpArg,  size_t nArg,
    const char  *cpVal,  size_t nVal,
          char **cppOut, size_t *pnOut, size_t *pnOutsize)
{
    char *cp;
    int msgcount;
    msg_t *msg;
    char *foreignid;
    char *nodename;
    time_t cachetime;
    pid_t cachepid;
    int size;
    int maxsize;
    char *format;

    if (nVal > 2 && cpVal[0] == '<' && cpVal[nVal-1] == '>') {
        *cppOut    = strdupex(cpVal);
        *pnOutsize = nVal;
        *pnOut     = nVal;
        return VAR_OK;
    }

    if (val_get(prival, "msgcount", &msgcount) != VAL_OK)
        msgcount = 0;

    if (val_get(prival, "msg", &msg) != VAL_OK)
        msg = NULL;

    if (msg != NULL && msg->cpFid != NULL)
        foreignid = msg->cpFid;
    else
        foreignid = "noFid";

    if (val_get(prival, "nodename", &nodename) != VAL_OK)
        nodename = "localhost.invalid";

    cachetime = time(NULL);
    cachepid = getpid();

    /* idea and first implementation as a contribution to lmtp2nntp v1.0.0 by Christos Ricudis <ricudis@paiko.gr> */

    maxsize = WRAPAT - strlen("Message-ID: ");
    format = "<%0.11d$%0.6d$%0.4d$%.20s@%s>";
    size = str_format(NULL, -1, format, cachetime, cachepid, msgcount, foreignid, nodename);
    if (size >= 38) {
        if (size > maxsize)
            size = maxsize;
        cp = (char *)mallocex(size + 1);
        size = str_format(cp, size + 1, format, cachetime, cachepid, msgcount, foreignid, nodename);
        if (size > maxsize)
            cp[size-2] = '>';
    }
    else
        cp = strdupex("<>"); /* if format fails, do not end the wholly program abnormally */

    *cppOut    = strdupex(cp);
    *pnOutsize = strlen(cp) + 1;
    *pnOut     = strlen(cp);
    return VAR_OK;
}

static var_rc_t operate_cb(
    var_t *var, void *_prival,
    const char  *op_ptr, size_t op_len,
    const char  *arg_ptr, size_t arg_len,
    const char  *val_ptr, size_t val_len,
    char **out_ptr, size_t *out_len, size_t *out_size)
{
    val_t *prival = _prival;

    if (op_len == 15 && strncmp(op_ptr, "createmessageid", 15) == 0) {
        return createmessageid(prival, arg_ptr, arg_len, val_ptr, val_len, out_ptr, out_len, out_size);
    }
    else
        return VAR_ERR_UNDEFINED_OPERATION;
}

void msg_headermatrixbuildup(msg_t *msg)
{
    ex_t ex;
    volatile headerdata_t *hdNew = NULL;
    try {
        headerdata_t *hdI, *hdP;
        char *cp;

        logbook(msg->l2, L2_LEVEL_TRACE, "headermatrix build up");

        cp = NULL;
        while ((cp = argz_next(msg->azHeaders, msg->asHeaders, cp)) != NULL) { /* for each message header */
            logbook(msg->l2, L2_LEVEL_DEBUG, "processing message header \"%s\"", cp);

            /*TODO we want O(1) here */
            for (hdP = NULL, hdI = msg->hdFirst; hdI != NULL; hdP = hdI, hdI = hdI->next) { /* for each matrix header */
                if (hdI->name == NULL || strlen(hdI->name) == 0 || hdI->ndata == 0)
                    continue;
                if (strcasecmp(cp, hdI->name) == 0)
                    break;
            }

            if (hdI == NULL) {
                hdNew = headercreate(); /* not found, create new */
                hdNew->name = strdupex(cp);
                hdI = (headerdata_t *)hdNew; hdNew = NULL; /* ex cleanup */
                if (hdP == NULL)
                    msg->hdFirst = hdI; /* no previous? this is the first */
                else {
                    hdP->next = hdI;
                    hdI->prev = hdP;
                }
            }
            cp = argz_next(msg->azHeaders, msg->asHeaders, cp);
            if (hdI->ndata == 0) {
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently empty", hdI->name);
                hdI->data.s = strdupex(cp);
                hdI->ndata = 1;
            }
            else if(hdI->ndata == 1) {
                char *cpOld;
                cpOld = hdI->data.s;
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently single valued", hdI->name);
                hdI->data.m = (char **)mallocex(3 * sizeof(char *));
                hdI->data.m[0] = cpOld;
                hdI->data.m[1] = strdupex(cp);
                hdI->data.m[2] = NULL;
                hdI->ndata = 2;
            }
            else {
                logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, currently multi valued %d", hdI->name, hdI->ndata);
                hdI->data.m = (char **)reallocex(hdI->data.m, (hdI->ndata + 2) * sizeof(char *));
                hdI->data.m[hdI->ndata++] = strdupex(cp);
                hdI->data.m[hdI->ndata] = NULL;
            }
        }
    }
    cleanup {
        if (hdNew != NULL)
            free((headerdata_t *)hdNew);
    }
    catch(ex) {
        rethrow;
    }
}

void msg_headermatrixteardwn(msg_t *msg)
{
    ex_t ex;

    logbook(msg->l2, L2_LEVEL_TRACE, "headermatrix tear down");

    try {
        headerdata_t *hdI;
        headerdata_t *hdN;

        if (msg->azHeaders != NULL)
            free(msg->azHeaders);
        msg->azHeaders = NULL;
        msg->asHeaders = 0;

        hdI = msg->hdFirst; /* for each header */
        while (hdI != NULL) {
            logbook(msg->l2, L2_LEVEL_DEBUG, "processing matrix header \"%s\"", hdI->name);
            if (hdI->name != NULL && strlen(hdI->name) != 0 && hdI->ndata != 0) {
                if (hdI->ndata == 0) {
                    logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, no data", hdI->name);
                }
                else if(hdI->ndata == 1) { /* header data is single valued */
                    logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s, data=%s", hdI->name, hdI->data.s);
                    argz_add(&msg->azHeaders, &msg->asHeaders, hdI->name);
                    argz_add(&msg->azHeaders, &msg->asHeaders, hdI->data.s);
                }
                else { /* header data is multi valued */
                    int i;
                    for (i = 0; i < hdI->ndata; i++) {
                        logbook(msg->l2, L2_LEVEL_DEBUG, "header=%s[%d], data=%s", hdI->name, i, hdI->data.m[i]);
                        argz_add(&msg->azHeaders, &msg->asHeaders, hdI->name);
                        argz_add(&msg->azHeaders, &msg->asHeaders, hdI->data.m[i]);
                    }
                }
            }
            /* cleanup */
            hdN = hdI->next;
            hdI->prev = NULL;
            hdI->next = NULL;
            headerdestroy(hdI);
            hdI = hdN;
        }
    }
    catch(ex) {
        rethrow;
    }
}

void headerrewrite(lmtp2nntp_t *ctx)
{
    headerrule_t *hrI;
    headerdata_t *hdI, *hdNew;
    regex_ctx_t *regex_ctx;
#define OVECSIZE 30
    int            ovec[OVECSIZE];
    var_rc_t var_rc;
    char *cp;

    /* short circuit in case no headerrules were set up */
    if (ctx->option_firstheaderrule == NULL)
        return;

#if 0
    { /* TODO debug only - wait for l2 to support code block bypassing */
        int i;
        headerrule_t *hrD;
        headerdata_t *hdD;

        logbook(ctx->l2, L2_LEVEL_DEBUG, "headerrewrite input");
        for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
            logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
        for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
            if (hdD->ndata == 0)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
            if (hdD->ndata == 1)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
            if (hdD->ndata > 1)
                for (i = 0; i < hdD->ndata; i++)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
        }
    }
#endif

    regex_ctx = (regex_ctx_t *)mallocex(sizeof(regex_ctx_t));
    regex_ctx->nMatch = 0;
    regex_ctx->acpMatch = NULL;
    regex_ctx->l2_env = ctx->l2_env;
    regex_ctx->l2 = ctx->l2;
    if ((var_rc = var_config(ctx->config_varregex, VAR_CONFIG_CB_VALUE, regex_lookup, regex_ctx)) != VAR_OK) {
        logbook(ctx->l2, L2_LEVEL_ERROR, "configure regex callback failed with %s (%d)", var_strerror(ctx->config_varregex, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
        throw(0,0,0);
    }
    if ((var_rc = var_config(ctx->config_varctx, VAR_CONFIG_CB_OPERATION, operate_cb, ctx->prival)) != VAR_OK) {
        logbook(ctx->l2, L2_LEVEL_ERROR, "configure operate callback failed with %s (%d)", var_strerror(ctx->config_varctx, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
        throw(0,0,0);
    }
    for (hrI = ctx->option_firstheaderrule; hrI != NULL; hrI = hrI->next) { /* for each rule */
#if 0
        { /* TODO debug only - wait for l2 to support code block bypassing */
            int i;
            headerrule_t *hrD;
            headerdata_t *hdD;

            logbook(ctx->l2, L2_LEVEL_DEBUG, "headerrewrite loop for each rule");
            for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
            for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
                /* logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD=%.8lx, hdD->name=%.8lx, hdD->data.s=%.8lx", (long)hdD, (long)&hdD->name, (long)&hdD->data.s); */
                if (hdD->ndata == 0)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
                if (hdD->ndata == 1)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
                if (hdD->ndata > 1)
                    for (i = 0; i < hdD->ndata; i++)
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
            }
        }
#endif
        if (hrI->regex != NULL) {
            logbook(ctx->l2, L2_LEVEL_DEBUG, "rule has regex %s", hrI->regex);
            for (hdI = ctx->msg->hdFirst; hdI != NULL; hdI = hdI->next) { /* for each header */
                if (hdI->name == NULL || strlen(hdI->name) == 0 || hdI->ndata == 0)
                    continue;
                regex_ctx->nMatch = pcre_exec(hrI->pcreRegex, hrI->pcreExtra, hdI->name, strlen(hdI->name), 0, 0, ovec, OVECSIZE);
                if (regex_ctx->nMatch >= 1) {
                    int i;
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "regex matches, %d references", regex_ctx->nMatch);
                    pcre_get_substring_list(hdI->name, ovec, regex_ctx->nMatch, &regex_ctx->acpMatch);
                    if (regex_ctx->acpMatch != NULL)
                        for (i = 0; i < regex_ctx->nMatch; i++)
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "regex reference[%d]=\'%s\'", i, regex_ctx->acpMatch[i] == NULL ? "(UNDEFINED)" : regex_ctx->acpMatch[i]);
                    hdNew = headercreate();
                    /* expanding regex references into header name */
                    {
                        char *res_ptr;
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding regex references in headername '%s'", hrI->name);
                        if ((var_rc = var_expand(ctx->config_varregex, hrI->name, strlen(hrI->name), &res_ptr, NULL, FALSE)) != VAR_OK) {
                            logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->name, var_strerror(ctx->config_varregex, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
                        }
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                        if (strlen(res_ptr) == 0) {
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - emtpy headername");
                            headerdatadestroy(hdNew);
                            headernamedestroy(hdNew);
                        }
                        else {
                            hdNew->name = res_ptr;
                        }
                    }
                    if (hrI->val == NULL) {
                        logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue before expansion");
                        headerdatadestroy(hdNew);
                        headernamedestroy(hdNew);
                    }
                    else {
                        /* expanding regex references into header value */
                        {
                            char *res_ptr;
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding regex references in header value '%s'", hrI->val);
                            if ((var_rc = var_expand(ctx->config_varregex, hrI->val, strlen(hrI->val), &res_ptr, NULL, FALSE)) != VAR_OK) {
                                logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->val, var_strerror(ctx->config_varregex, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
                            }
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                            cp = res_ptr;
                        }
                        /* expanding variables into header value */
                        if (hrI->val != NULL) {
                            char *res_ptr;
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding variables in header value '%s'", hrI->val);
                            if ((var_rc = var_expand(ctx->config_varctx, cp, strlen(cp), &res_ptr, NULL, FALSE)) != VAR_OK) {
                                logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", cp, var_strerror(ctx->config_varctx, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
                            }
                            logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                            if (strlen(res_ptr) == 0) {
                                logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue after expansion");
                                headerdatadestroy(hdNew);
                                headernamedestroy(hdNew);
                            }
                            else {
                                hdNew->data.s = res_ptr;
                                hdNew->ndata = 1;
                            }
                        }
                    }
                    headerreplace(hdI, hdNew);
                    if (hdNew->prev == NULL)
                        ctx->msg->hdFirst = hdNew;
                    hdI = hdNew;
                }
            }
        }
        else {
            logbook(ctx->l2, L2_LEVEL_DEBUG, "rule has no regex but static header %s", hrI->name);
            hdNew = headercreate();
            hdNew->name = strdupex(hrI->name);
            if (hrI->val == NULL) {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted");
                headerdatadestroy(hdNew);
                headernamedestroy(hdNew);
            }
            else {
                /* expanding variables into header value */
                char *res_ptr;
                logbook(ctx->l2, L2_LEVEL_DEBUG, "expanding variables in header value '%s'", hrI->val);
                if ((var_rc = var_expand(ctx->config_varctx, hrI->val, strlen(hrI->val), &res_ptr, NULL, FALSE)) != VAR_OK) {
                    logbook(ctx->l2, L2_LEVEL_ERROR, "expansion of '%s' failed: %s", hrI->val, var_strerror(ctx->config_varctx, var_rc, &cp) == VAR_OK ? cp : "Unknown Error", var_rc);
                }
                logbook(ctx->l2, L2_LEVEL_DEBUG, "expansion result '%s'", res_ptr);
                if (strlen(res_ptr) == 0) {
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "marking deleted - empty headervalue after expansion");
                    headerdatadestroy(hdNew);
                    headernamedestroy(hdNew);
                }
                else {
                    headerdatadestroy(hdNew);
                    hdNew->data.s = res_ptr;
                    hdNew->ndata = 1;
                }
            }
            for (hdI = ctx->msg->hdFirst; hdI != NULL; hdI = hdI->next) { /* for each header */
                if (hdI->name == NULL || strlen(hdI->name) == 0)
                    continue;
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hrI->name=%s, hdI->name=%s", hrI->name, hdI->name);
                if (strcasecmp(hrI->name, hdI->name) == 0)
                    break;
            }
            if (hdI != NULL) {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "replacing header %s", hrI->name);
                headerreplace(hdI, hdNew);
                if (hdNew->prev == NULL)
                    ctx->msg->hdFirst = hdNew;
            }
            else {
                logbook(ctx->l2, L2_LEVEL_DEBUG, "appending header %s", hrI->name);
                for (hdI = ctx->msg->hdFirst; hdI->next != NULL; hdI = hdI->next);
                hdI->next = hdNew;
                hdNew->prev = hdI;
            }
        }
    }

#if 0
    { /* TODO debug only - wait for l2 to support code block bypassing */
        int i;
        headerrule_t *hrD;
        headerdata_t *hdD;

        logbook(ctx->l2, L2_LEVEL_DEBUG, "headerrewrite output");
        for (hrD = ctx->option_firstheaderrule; hrD != NULL; hrD = hrD->next)
            logbook(ctx->l2, L2_LEVEL_DEBUG, "hrD->header=%s", hrD->header);
        for (hdD = ctx->msg->hdFirst; hdD != NULL; hdD = hdD->next) {
            if (hdD->ndata == 0)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name=%s: (NO DATA)", hdD->name);
            if (hdD->ndata == 1)
                logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.s    %s %s", hdD->name, hdD->data.s);
            if (hdD->ndata > 1)
                for (i = 0; i < hdD->ndata; i++)
                    logbook(ctx->l2, L2_LEVEL_DEBUG, "hdD->name:hdD->data.m[%d] %s %s", i, hdD->name, hdD->data.m[i]);
        }
    }
#endif
}
