/***************************************************************************
 *   Copyright (C) 2008 by S. MANKOWSKI / G. DE BURE support@mankowski.fr  *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>  *
 ***************************************************************************/
/** @file
* This file defines classes SKGImportExportManager.
*
* @author Stephane MANKOWSKI / Guillaume DE BURE
 */
#include "skgimportexportmanager.h"

#include <klocalizedstring.h>
#include <kservicetypetrader.h>

#include <qcryptographichash.h>
#include <qfileinfo.h>
#include <qregexp.h>
#include <qtemporaryfile.h>
#include <qtextcodec.h>

#include <algorithm>

#include "skgbankincludes.h"
#include "skgimportplugin.h"
#include "skgobjectbase.h"
#include "skgpayeeobject.h"
#include "skgrecurrentoperationobject.h"
#include "skgruleobject.h"
#include "skgservices.h"
#include "skgtraces.h"

SKGImportExportManager::SKGImportExportManager(SKGDocumentBank* iDocument,
        const QUrl& iFileName)
    : QObject(), m_document(iDocument), m_fileName(iFileName),
      m_defaultAccount(nullptr), m_defaultUnit(nullptr),
      m_importPlugin(nullptr), m_exportPlugin(nullptr)
{
    setAutomaticValidation(true);
    setAutomaticApplyRules(false);
    m_since_last_import = true;
    m_codec = QStringLiteral("UTF-8");
}

SKGImportExportManager::~SKGImportExportManager()
{
    setDefaultAccount(nullptr);
    setDefaultUnit(nullptr);
    m_document = nullptr;
    m_defaultAccount = nullptr;
    m_defaultUnit = nullptr;
    m_importPlugin = nullptr;
    m_exportPlugin = nullptr;

    if (!m_localFileName.isEmpty() && m_localFileName != getFileName().toLocalFile()) {
        QFile(m_localFileName).remove();
    }
}

QString SKGImportExportManager::getFileNameExtension() const
{
    return QFileInfo(getFileName().path()).suffix().toUpper();
}

QUrl SKGImportExportManager::getFileName() const
{
    return m_fileName;
}

QString SKGImportExportManager::getLocalFileName(bool iDownload)
{
    if (m_localFileName.isEmpty()) {
        if (getFileName().isLocalFile()) {
            m_localFileName = getFileName().toLocalFile();
        } else {
            if (iDownload) {
                SKGServices::download(QUrl(getFileName()), m_localFileName);
            } else {
                QTemporaryFile tmpFile;
                tmpFile.setAutoRemove(false);
                tmpFile.open();
                m_localFileName = tmpFile.fileName();
            }
        }
    }
    return m_localFileName;
}

SKGError SKGImportExportManager::setDefaultAccount(SKGAccountObject* iAccount)
{
    SKGError err;
    SKGTRACEINFUNCRC(10, err);
    delete m_defaultAccount;
    m_defaultAccount = nullptr;
    if (iAccount) {
        m_defaultAccount = new SKGAccountObject(*iAccount);
    }
    return err;
}

SKGError SKGImportExportManager::getDefaultAccount(SKGAccountObject& oAccount)
{
    SKGError err;
    SKGTRACEINFUNCRC(10, err);
    if (m_defaultAccount == nullptr && m_document) {
        QFileInfo fInfo(getLocalFileName());
        QStringList items = fInfo.baseName().split('@');
        QString bankname;
        QString number;
        QString name = items.at(0);
        if (items.count() == 2) {
            bankname = items.at(1);
        } else {
            bankname = name;
        }
        name.replace('_', ' ');
        name.replace('-', ' ');
        QRegExp rx(QStringLiteral("(\\d{6,})"));
        if (rx.indexIn(name) != -1) {
            number = rx.cap(1);
            if (name != number) {
                name = name.remove(number).trimmed();
            }
        }

        // Searching if an account exist
        QString whereClause = "t_name='" % name % '\'';
        if (!number.isEmpty()) {
            whereClause += " OR t_number='" % number % "'";
            whereClause += " OR t_agency_number||t_number='" % number % "'";
            whereClause += " OR t_BANK_NUMBER||t_agency_number||t_number='" % number % "'";
            whereClause += " OR '" % number % "' LIKE t_BANK_NUMBER||t_agency_number||t_number||'__'";
        }
        const auto words = name.split(' ');
        for (const auto& val : words) {
            whereClause += " OR t_number='" % val % "'";
        }

        SKGObjectBase::SKGListSKGObjectBase listAccount;
        err = m_document->getObjects(QStringLiteral("v_account"), whereClause % " ORDER BY t_type ASC", listAccount);
        IFOK(err) {
            if (!listAccount.isEmpty()) {
                // Yes ! Only one account found
                SKGAccountObject account(listAccount.at(0));
                m_defaultAccount = new SKGAccountObject(account);
                err = m_document->sendMessage(i18nc("An information message",  "Using account '%1' for import", account.getName()));
            }
        }

        // If better account not found, then we must create one
        if (m_defaultAccount == nullptr) {
            SKGAccountObject account;
            SKGBankObject bank(m_document);
            IFOKDO(err, bank.setName(bankname))
            if (!err && bank.load().isFailed()) {
                err = bank.save(false);    // Save only
            }
            IFOKDO(err, bank.addAccount(account))
            IFOKDO(err, account.setName(name))
            IFOKDO(err, account.setNumber(number))
            if (!err && account.load().isFailed()) {
                err = account.save(false);    // Save only
            }
            IFOK(err) m_defaultAccount = new SKGAccountObject(account);
            IFOKDO(err, m_document->sendMessage(i18nc("An information message",  "Default account '%1' created for import", name)))
        }
    }

    if (m_defaultAccount != nullptr) {
        oAccount = *m_defaultAccount;
    }

    return err;
}

SKGError SKGImportExportManager::setDefaultUnit(SKGUnitObject* iUnit)
{
    SKGError err;
    SKGTRACEINFUNCRC(10, err);
    delete m_defaultUnit;
    m_defaultUnit = nullptr;
    if (iUnit) {
        m_defaultUnit = new SKGUnitObject(*iUnit);
    }
    return err;
}

SKGError SKGImportExportManager::getDefaultUnit(SKGUnitObject& oUnit, const QDate* iDate)
{
    SKGError err;
    SKGTRACEINFUNCRC(10, err);
    if (m_document && (m_defaultUnit == nullptr || iDate)) {
        if (m_defaultUnit == nullptr) {
            delete m_defaultUnit;
            m_defaultUnit = nullptr;
        }

        // Do we have to found the best unit for a date ?
        QString wc = QStringLiteral("t_type IN ('1', '2', 'C')");
        if (iDate) {
            // Yes
            wc += " AND (d_MINDATE<'" % SKGServices::dateToSqlString(QDateTime(*iDate)) % "' OR d_MINDATE IS NULL)";
        }

        // Check if a unit exist
        SKGObjectBase::SKGListSKGObjectBase listUnits;
        err = m_document->getObjects(QStringLiteral("v_unit"), wc % " ORDER BY ABS(f_CURRENTAMOUNT-1) ASC" , listUnits);
        IFOK(err) {
            if (listUnits.isEmpty()) {
                SKGUnitObject unit(m_document);
                QString name = i18nc("Noun",  "Unit for import");
                err = unit.setName(name);
                if (unit.load().isFailed()) {
                    IFOKDO(err, unit.setSymbol(name))
                    IFOKDO(err, unit.save(false))

                    SKGUnitValueObject unitval;
                    IFOKDO(err, unit.addUnitValue(unitval))
                    IFOKDO(err, unitval.setQuantity(1))
                    IFOKDO(err, unitval.setDate(QDate(1970, 1, 1)))
                    IFOKDO(err, unitval.save(false, false))

                    IFOKDO(err, m_document->sendMessage(i18nc("An information message",  "Default unit '%1' created for import", name)))
                }

                IFOK(err) m_defaultUnit = new SKGUnitObject(unit);
            } else {
                // Found, we can use it
                m_defaultUnit = new SKGUnitObject(listUnits.at(0));
            }
        }
    }

    if (m_defaultUnit != nullptr) {
        oUnit = *m_defaultUnit;
    }

    return err;
}

void SKGImportExportManager::setAutomaticValidation(bool iValidation)
{
    m_automaticValidationOfImportedOperation = iValidation;
}

bool SKGImportExportManager::automaticValidation() const
{
    return m_automaticValidationOfImportedOperation;
}

void SKGImportExportManager::setAutomaticApplyRules(bool iValidation)
{
    m_automaticApplyRulesOfImportedOperation = iValidation;
}

bool SKGImportExportManager::automaticApplyRules() const
{
    return m_automaticApplyRulesOfImportedOperation;
}

void SKGImportExportManager::setSinceLastImportDate(bool iSinceLast)
{
    m_since_last_import = iSinceLast;
}

bool SKGImportExportManager::sinceLastImportDate() const
{
    return m_since_last_import;
}

void SKGImportExportManager::addAccountToCheck(const SKGAccountObject& iAccount, double iBalance)
{
    m_AccountToCheck.push_back(iAccount);
    m_BalanceToCheck.push_back(iBalance);
}

SKGError SKGImportExportManager::finalizeImportation()
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);
    if (m_document && !getImportParameters().contains(QStringLiteral("donotfinalize"))) {
        // Count the number of operations imported but already existing
        QString wc = "v_operation.t_imported='T' AND exists (SELECT 1 from v_operation op2 WHERE op2.t_imported!='T' AND op2.rd_account_id=v_operation.rd_account_id AND op2.t_import_id=v_operation.t_import_id AND ABS(op2.f_CURRENTAMOUNT-v_operation.f_CURRENTAMOUNT)<" % SKGServices::doubleToString(EPSILON) % ')';
        int nbOperations;
        err = m_document->getNbObjects(QStringLiteral("v_operation"), wc, nbOperations);
        if (!err && nbOperations) {
            err = m_document->sendMessage(i18np("One operation not imported because it already exists", "%1 operations not imported because they already exist", nbOperations), SKGDocument::Warning);

            // Remove operations imported for nothing
            IFOKDO(err, m_document->executeSqliteOrder("DELETE from operation WHERE id IN (SELECT id from v_operation WHERE " % wc % ')'))
        }

        // Delete operations before the last import
        if (sinceLastImportDate() && !err) {
            SKGTRACEINFUNCRC(2, err);

            // Count the number of operations before the last import
            QString wc2 = QStringLiteral("operation.t_imported='T' AND "
                                         "EXISTS(SELECT 1 FROM operation o INDEXED BY idx_operation_rd_account_id_t_imported WHERE o.rd_account_id=operation.rd_account_id AND o.t_imported IN ('Y', 'P') AND date(o.d_date,'-4 day')>operation.d_date)");
            int nbOperations2;
            err = m_document->getNbObjects(QStringLiteral("operation"), wc2, nbOperations2);
            if (!err && nbOperations2) {
                err = m_document->sendMessage(i18np("One operation was not imported because it was dated before the last imported one, you can uncheck the option to avoid this.", "%1 operations were not imported because they were dated before the last imported one, you can uncheck the option to avoid this.", nbOperations2), SKGDocument::Warning);

                // Remove operations imported for nothing
                IFOKDO(err, m_document->executeSqliteOrder("DELETE from operation WHERE " % wc2))
            }
        }

        // For performances
        IFOKDO(err, m_document->executeSqliteOrder(QStringLiteral("ANALYZE")))

        // Apply rules
        if (!err && m_automaticApplyRulesOfImportedOperation) {
            SKGTRACEINFUNCRC(2, err);
            // Get rules
            SKGObjectBase::SKGListSKGObjectBase rules;
            IFOKDO(err, m_document->getObjects(QStringLiteral("v_rule"), QStringLiteral("1=1 ORDER BY f_sortorder"), rules))

            int nbRules = rules.count();
            IFOKDO(err, m_document->beginTransaction("#INTERNAL#" % i18nc("Progression step", "Finalize import"), nbRules))
            for (int i = 0; !err && i < nbRules; ++i) {
                SKGRuleObject rule(rules.at(i));
                err = rule.execute(SKGRuleObject::IMPORTING);
                IFOKDO(err, m_document->stepForward(i + 1))
            }
            SKGENDTRANSACTION(m_document,  err);

            // A failure in rule will not block the transaction.
            // A warning message is sent
            IFKO(err) err = m_document->sendMessage(i18nc("Warning message", "Error during execution of rules:\n%1", err.getFullMessageWithHistorical()), SKGDocument::Error);
        }

        // Change imported status
        IFOK(err) {
            SKGTRACEINFUNCRC(2, err);
            err = m_document->executeSqliteOrder(QStringLiteral("UPDATE operation SET t_imported='") %
                                                 (m_automaticValidationOfImportedOperation ? QStringLiteral("Y") : QStringLiteral("P")) %
                                                 "' WHERE t_imported='T'");
        }

        // Check balances of accounts
        int nb = m_AccountToCheck.count();
        for (int i = 0; !err && i < nb; ++i) {
            // Get the account to check
            auto act = m_AccountToCheck.at(i);
            auto targetBalance = m_BalanceToCheck.at(i);
            err = act.setAttribute(QStringLiteral("f_importbalance"), SKGServices::doubleToString(targetBalance));
            IFOKDO(err, act.save());

            auto soluces = act.getPossibleReconciliations(targetBalance, false);
            if (soluces.isEmpty()) {
                IFOKDO(err, m_document->sendMessage(i18nc("Information message", "The balance of account '%1' is not aligned with import balance %2", act.getDisplayName(), m_document->formatMoney(targetBalance, m_document->getPrimaryUnit())), SKGDocument::Warning));
            } else {
                IFOKDO(err, m_document->sendMessage(i18nc("Information message", "The balance of account '%1' is aligned with import balance %2", act.getDisplayName(), m_document->formatMoney(targetBalance, m_document->getPrimaryUnit())), SKGDocument::Positive));
            }
        }
    }
    return err;
}

void SKGImportExportManager::setImportParameters(const QMap< QString, QString >& iParameters)
{
    SKGImportPlugin* plugin = getImportPlugin();
    if (plugin) {
        plugin->setImportParameters(iParameters);
    }
}

QMap< QString, QString > SKGImportExportManager::getImportParameters()
{
    QMap< QString, QString > output;
    SKGImportPlugin* plugin = getImportPlugin();
    if (plugin) {
        output = plugin->getImportParameters();
    }
    return output;
}

QString SKGImportExportManager::getImportMimeTypeFilter(bool iIncludingAll)
{
    QMap<QString, QString> tmp;
    KService::List offers = KServiceTypeTrader::self()->query(QStringLiteral("SKG IMPORT/Plugin"));
    int nb = offers.count();
    for (int i = 0; i < nb; ++i) {
        KService::Ptr service = offers.at(i);
        KPluginLoader loader(service->library());
        KPluginFactory* factory = loader.factory();
        if (factory) {
            SKGImportPlugin* pluginInterface = factory->create<SKGImportPlugin> (0);
            if (pluginInterface && pluginInterface->isImportPossible()) {
                QString mime = pluginInterface->getMimeTypeFilter();
                if (!mime.isEmpty()) {
                    QStringList lines = SKGServices::splitCSVLine(mime, '\n');
                    int nblines = lines.count();
                    for (int l = 0; l < nblines; ++l) {
                        QStringList items = SKGServices::splitCSVLine(lines.at(l), '|');
                        tmp[items.at(1)] = items.at(0);
                    }
                }
            }
        }
    }

    QStringList descriptions = tmp.keys();
    std::sort(descriptions.begin(), descriptions.end());
    QStringList regexps = tmp.values();

    QString output;
    if (iIncludingAll) {
        output = regexps.join(QStringLiteral(" ")) % '|' % i18nc("A file format", "All supported formats");
    }
    nb = descriptions.count();
    for (int i = 0; i < nb; ++i) {
        if (!output.isEmpty()) {
            output += '\n';
        }
        output += tmp[descriptions.at(i)] % '|' % descriptions.at(i);
    }

    return output;
}

QString SKGImportExportManager::getExportMimeTypeFilter(bool iIncludingAll)
{
    QMap<QString, QString> tmp;
    KService::List offers = KServiceTypeTrader::self()->query(QStringLiteral("SKG IMPORT/Plugin"));
    int nb = offers.count();
    for (int i = 0; i < nb; ++i) {
        KService::Ptr service = offers.at(i);
        KPluginLoader loader(service->library());
        KPluginFactory* factory = loader.factory();
        if (factory) {
            SKGImportPlugin* pluginInterface = factory->create<SKGImportPlugin> (0);
            if (pluginInterface && pluginInterface->isExportPossible()) {
                QString mime = pluginInterface->getMimeTypeFilter();
                if (!mime.isEmpty()) {
                    QStringList lines = SKGServices::splitCSVLine(mime, '\n');
                    int nblines = lines.count();
                    for (int l = 0; l < nblines; ++l) {
                        QStringList items = SKGServices::splitCSVLine(lines.at(l), '|');
                        tmp[items.at(1)] = items.at(0);
                    }
                }
            }
        }
    }

    QStringList descriptions = tmp.keys();
    std::sort(descriptions.begin(), descriptions.end());
    QStringList regexps = tmp.values();

    QString output;
    if (iIncludingAll) {
        output = regexps.join(QStringLiteral(" ")) % '|' % i18nc("A file format", "All supported formats");
    }
    nb = descriptions.count();
    for (int i = 0; i < nb; ++i) {
        if (!output.isEmpty()) {
            output += '\n';
        }
        output += tmp[descriptions.at(i)] % '|' % descriptions.at(i);
    }

    return output;
}


SKGImportPlugin* SKGImportExportManager::getImportPlugin()
{
    if (!m_importPlugin) {
        KService::List offers = KServiceTypeTrader::self()->query(QStringLiteral("SKG IMPORT/Plugin"));
        int nb = offers.count();
        for (int i = 0; !m_importPlugin && i < nb; ++i) {
            KService::Ptr service = offers.at(i);
            QString id = service->property(QStringLiteral("X-Krunner-ID"), QVariant::String).toString();
            KPluginLoader loader(service->library());
            KPluginFactory* factory = loader.factory();
            if (factory) {
                SKGImportPlugin* pluginInterface = factory->create<SKGImportPlugin> (this);
                if (pluginInterface && pluginInterface->isImportPossible()) {
                    // Import
                    m_importPlugin = pluginInterface;
                }
            } else if (m_document) {
                m_document->sendMessage(i18nc("An information message",  "Loading plugin %1 failed because the factory could not be found in %2", id, service->library()), SKGDocument::Error);
            }
        }
    }
    return m_importPlugin;
}

SKGImportPlugin* SKGImportExportManager::getExportPlugin()
{
    if (!m_exportPlugin) {
        KService::List offers = KServiceTypeTrader::self()->query(QStringLiteral("SKG IMPORT/Plugin"));
        int nb = offers.count();
        for (int i = 0; !m_exportPlugin && i < nb; ++i) {
            KService::Ptr service = offers.at(i);
            QString id = service->property(QStringLiteral("X-Krunner-ID"), QVariant::String).toString();
            KPluginLoader loader(service->library());
            KPluginFactory* factory = loader.factory();
            if (factory) {
                SKGImportPlugin* pluginInterface = factory->create<SKGImportPlugin> (this);
                if (pluginInterface && pluginInterface->isExportPossible()) {
                    // Import
                    m_exportPlugin = pluginInterface;
                }
            } else if (m_document) {
                m_document->sendMessage(i18nc("An information message",  "Loading plugin %1 failed because the factory could not be found in %2", id, service->library()), SKGDocument::Error);
            }
        }
    }
    return m_exportPlugin;
}

SKGError SKGImportExportManager::importFile()
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);
    if (m_document) {
        SKGBEGINPROGRESSTRANSACTION(*m_document, i18nc("Noun, name of the user action", "Import with codec %1", m_codec), err, 3);

        err = m_document->executeSqliteOrder(QStringLiteral("ANALYZE"));

        IFOKDO(err, m_document->stepForward(1))

        IFOK(err) {
            // Search plugins
            bool fileTreated = false;
            SKGImportPlugin* pluginInterface = getImportPlugin();
            if (pluginInterface) {
                // Import
                fileTreated = true;
                m_AccountToCheck.clear();
                m_BalanceToCheck.clear();
                SKGTRACEL(2) << "Input filename=" << m_fileName.toDisplayString() << endl;
                SKGTRACEL(2) << "Input local filename=" << getLocalFileName() << endl;
                err = pluginInterface->importFile();
            }

            if (!err && !fileTreated) {
                err.setReturnCode(ERR_NOTIMPL).setMessage(i18nc("Error message",  "This import mode is not yet implemented"));
            }
        }
        IFOKDO(err, m_document->stepForward(2))

        IFOKDO(err, finalizeImportation())
        IFOKDO(err, m_document->stepForward(3))
    }

    return err;
}

SKGError SKGImportExportManager::exportFile()
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);
    if (m_document) {
        err = m_document->executeSqliteOrder(QStringLiteral("ANALYZE"));
        IFOK(err) {
            // Search plugins
            bool fileTreated = false;
            SKGImportPlugin* pluginInterface = getExportPlugin();
            if (pluginInterface) {
                // Import
                fileTreated = true;
                SKGTRACEL(2) << "Input filename=" << m_fileName.toDisplayString() << endl;
                SKGTRACEL(2) << "Input local filename=" << getLocalFileName(false) << endl;
                err = pluginInterface->exportFile();
                IFOKDO(err, SKGServices::upload(QUrl::fromLocalFile(getLocalFileName(false)), m_fileName))
            }

            if (!err && !fileTreated) {
                err.setReturnCode(ERR_NOTIMPL).setMessage(i18nc("Error message",  "This export mode is not yet implemented"));
            }
        }
    }

    return err;
}

SKGError SKGImportExportManager::cleanBankImport()
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);

    // Begin transaction
    if (m_document) {
        err = m_document->beginTransaction("#INTERNAL#" % i18nc("Progression step", "Clean import"), 3);
        IFOK(err) {
            // Step 1 Clean operations without mode and with comment with double space
            SKGObjectBase::SKGListSKGObjectBase operations;
            IFOKDO(err, m_document->getObjects(QStringLiteral("operation"), QStringLiteral("t_imported!='N' and t_mode='' and t_comment like '%  %'"), operations))

            int nb = operations.count();
            for (int i = 0; !err && i < nb ; ++i) {
                SKGOperationObject op(operations.at(i));

                // Comment is like this: <TYPE>  <INFO>
                // Example: RETRAIT DAB             20/01/08 11H44 013330 LCL GAILLAC 000497
                QRegExp rx(QStringLiteral("(.+) {2,}(.+)"));
                QString comment = op.getComment();
                if (rx.indexIn(comment) != -1) {
                    // Get parameters
                    QString mode = rx.cap(1);
                    QString info = rx.cap(2);

                    // Modify
                    err = op.setComment(info.trimmed());
                    IFOKDO(err, op.setMode(mode.trimmed()))
                    IFOKDO(err, op.save(true, false))   // No reload
                }
            }

            // Step 1 done
            IFOKDO(err, m_document->stepForward(1))

            // Step 2 Clean operations without mode and with comment
            IFOKDO(err, m_document->getObjects(QStringLiteral("operation"), QStringLiteral("t_imported!='N' and t_mode='' and t_comment!=''"), operations))

            nb = operations.count();
            for (int i = 0; !err && i < nb ; ++i) {
                SKGOperationObject op(operations.at(i));

                // Comment is like this: <TYPE> <INFO>
                // Example: RETRAIT DAB 14-05-16607-482390
                QRegExp rx(QStringLiteral("(\\S+) +(.+)"));
                QString comment = op.getComment();
                if (rx.indexIn(comment) != -1) {
                    // Get parameters
                    QString mode = rx.cap(1);
                    QString info = rx.cap(2);

                    // Modify
                    err = op.setComment(info.trimmed());
                    IFOKDO(err, op.setMode(mode.trimmed()))
                    IFOKDO(err, op.save(true, false))   // No reload
                }
            }

            // Step 2 done
            IFOKDO(err, m_document->stepForward(2))

            // Step 3 Clean cheque without number
            IFOKDO(err, m_document->getObjects(QStringLiteral("operation"), QStringLiteral("t_imported!='N' and i_number=0 and lower(t_mode)='cheque'"), operations))

            nb = operations.count();
            for (int i = 0; !err && i < nb ; ++i) {
                SKGOperationObject op(operations.at(i));

                // Comment is like this: <TYPE>  <INFO>
                // Example: RETRAIT DAB             20/01/08 11H44 013330 LCL GAILLAC 000497
                QRegExp rx(QStringLiteral("(\\d+)"));
                QString comment = op.getComment();
                if (rx.indexIn(comment) != -1) {
                    // Get parameters
                    int number = SKGServices::stringToInt(rx.cap(1));

                    // Modify
                    err = op.setNumber(number);
                    IFOKDO(err, op.save(true, false))   // No reload
                }
            }

            // Step 3 done
            IFOKDO(err, m_document->stepForward(3))
        }

        SKGENDTRANSACTION(m_document,  err);
    }

    return err;
}

SKGError SKGImportExportManager::anonymize()
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);
    if (m_document) {
        if (m_document->isFileModified()) {
            err = SKGError(ERR_ABORT, i18nc("An information message",  "The document must be saved to be anonymized."), QStringLiteral("skg://file_save"));
        } else {
            {
                // Anonymize data
                QStringList sqlOrders;
                sqlOrders << QStringLiteral("UPDATE bank SET t_bank_number='', t_name='bank_'||id")
                          << QStringLiteral("UPDATE account SET t_number='', t_agency_number='', t_agency_address='', t_comment='', t_name='account_'||id")
                          << QStringLiteral("UPDATE category SET t_name='category_'||id")
                          << QStringLiteral("UPDATE payee SET t_address='', t_name='payee_'||id")
                          << QStringLiteral("UPDATE refund SET t_comment='', t_name='tracker_'||id")
                          << QStringLiteral("UPDATE operation SET t_comment=''")
                          << QStringLiteral("UPDATE suboperation SET t_comment='', f_value=f_value%1234.56")
                          << QStringLiteral("DELETE FROM parameters WHERE t_name NOT LIKE 'SKG_%' OR t_name='SKG_PASSWORD'")
                          << QStringLiteral("DELETE FROM doctransactionitem");  // For better performances

                int nb = sqlOrders.count();
                SKGBEGINPROGRESSTRANSACTION(*m_document, "##INTERNAL##" % i18nc("Progression step", "Anonymize"), err, nb);
                for (int i = 0; !err && i < nb; ++i) {
                    err = m_document->executeSqliteOrder(sqlOrders.at(i));
                    IFOKDO(err, m_document->stepForward(i + 1))
                }
            }

            // Remove transactions
            IFOKDO(err, m_document->removeAllTransactions())

            // Save new file
            QString newName = m_document->getCurrentFileName().replace(QStringLiteral(".skg"), QStringLiteral("-anonymized.skg"));
            if (newName == m_document->getCurrentFileName()) {
                newName = m_document->getCurrentFileName() % "-anonymized.skg";
            }
            IFOKDO(err, m_document->saveAs(newName, true))
        }
    }
    return err;
}

SKGError SKGImportExportManager::findAndGroupTransfers(int& oNbOperationsMerged, const QString& iAdditionnalCondition)
{
    SKGError err;
    SKGTRACEINFUNCRC(2, err);

    oNbOperationsMerged = 0;

    // Begin transaction
    if (m_document) {
        err = m_document->beginTransaction("#INTERNAL#" % i18nc("Progression step", "Find and group transfers"), 3);
        IFOK(err) {
            IFOKDO(err, m_document->executeSqliteOrder(QStringLiteral("ANALYZE")))
            // Look for operations with
            //  Same units
            //  Same dates
            //  Null i_group_id
            //  Different accounts
            //  Oposite amounts
            SKGStringListList listTmp;
            IFOKDO(err, m_document->executeSelectSqliteOrder(
                       "SELECT A.id, B.id FROM v_operation_tmp1 A, v_operation_tmp1 B WHERE +A.d_date=B.d_date AND A.rc_unit_id=B.rc_unit_id AND A.id<=B.id AND A.rd_account_id!=B.rd_account_id AND ABS(A.f_QUANTITY+B.f_QUANTITY)<" % SKGServices::doubleToString(EPSILON) % " AND A.i_group_id=0 AND B.i_group_id=0 AND A.f_QUANTITY!=0" %
                       (iAdditionnalCondition.isEmpty() ? QString() : QStringLiteral(" AND ") % iAdditionnalCondition),
                       listTmp));
            // Step 1 done
            IFOKDO(err, m_document->stepForward(1))

            SKGStringListList listTmp2;
            // +A.i_group_id=0 AND +B.i_group_id=0 is for avoiding to use bad index
            IFOKDO(err, m_document->executeSelectSqliteOrder(
                       "SELECT A.id, B.id FROM v_operation A, v_operation_tmp1 B, parameters P WHERE +P.t_uuid_parent=B.id||'-operation' AND A.rc_unit_id!=B.rc_unit_id AND P.t_name='SKG_OP_ORIGINAL_AMOUNT' AND A.d_date=B.d_date AND A.rd_account_id!=B.rd_account_id AND ABS(A.f_CURRENTAMOUNT+CAST(P.t_value AS REAL))<" % SKGServices::doubleToString(EPSILON) % " AND +A.i_group_id=0 AND +B.i_group_id=0 AND A.f_CURRENTAMOUNT!=0" %
                       (iAdditionnalCondition.isEmpty() ? QString() : QStringLiteral(" AND ") % iAdditionnalCondition),
                       listTmp2));

            // Step 2 done
            IFOKDO(err, m_document->stepForward(2))
            listTmp2.removeAt(0);    // Remove header
            listTmp += listTmp2;

            // Group
            {
                oNbOperationsMerged = listTmp.count();
                IFOKDO(err, m_document->beginTransaction("#INTERNAL#" % i18nc("Progression step", "Find and group transfers"), oNbOperationsMerged - 1))
                for (int i = 1; !err && i < oNbOperationsMerged ; ++i) {  // First line ignored because of its header
                    SKGOperationObject op1(m_document, SKGServices::stringToInt(listTmp.at(i).at(0)));
                    SKGOperationObject op2(m_document, SKGServices::stringToInt(listTmp.at(i).at(1)));

                    if (!op1.isInGroup() && !op2.isInGroup()) {
                        err = op2.setGroupOperation(op1);
                        IFOKDO(err, op2.save(true, false))
                    }
                    IFOKDO(err, m_document->stepForward(i))
                }
                SKGENDTRANSACTION(m_document,  err);
            }
            oNbOperationsMerged = (oNbOperationsMerged - 1) * 2;

            // Step 3 done
            IFOKDO(err, m_document->stepForward(3))
        }
        SKGENDTRANSACTION(m_document,  err);
    }

    return err;
}

SKGError SKGImportExportManager::findAndGroupTransfers(int& oNbOperationsMerged, bool iOnCurrentlyImport)
{
    return findAndGroupTransfers(oNbOperationsMerged, iOnCurrentlyImport ? QStringLiteral("A.t_imported='T' AND B.t_imported='T'") : QStringLiteral(""));
}


