/*
 * This file is part of the FORS Data Reduction Pipeline
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <string>
#include <sstream>
#include <cmath>
#include <vector>
#include <fors_bias_impl.h>

#include <fors_stack.h>
#include <fors_header.h>
#include <fors_tools.h>
#include <fors_dfs.h>
#include <fors_utils.h>
#include <fors_bpm.h>
#include <moses.h>
#include "fors_ccd_config.h"
#include "mosca_image.h"
#include "statistics.h"
#include "fors_overscan.h"
#include "fors_detmodel.h"

#include <cpl.h>

#include <string.h>
#include <math.h>

/**
 * @addtogroup fors_bias
 */

/**@{*/

const char *const fors_bias_name = "fors_bias";
const char *const fors_bias_description_short = "Compute the master bias frame";
const char *const fors_bias_author = "Jonas M. Larsen, Carlo Izzo";
const char *const fors_bias_email = PACKAGE_BUGREPORT;
const char *const fors_bias_description =
"This recipe is used to combine input raw BIAS frames into a master bias\n"
"frame. The overscan regions, if present, are removed from the result.\n\n"
"Input files:\n\n"
"  DO category:               Type:       Explanation:         Required:\n"
"  BIAS                       Raw         Bias frame              Y\n\n"
"Output files:\n\n"
"  DO category:               Data type:  Explanation:\n"
"  MASTER_BIAS                FITS image  Master bias frame\n\n";


void fors_bias_compute_ron
(fors_image_list * bias_raw, mosca::ccd_config& bias_ccd_config);

static void
write_qc(cpl_propertylist *qc,
         const fors_setting *setting,
         const cpl_frame *first_bias,
         const fors_image_list *bias,
         const fors_image *master_bias,
         const stack_method *sm,
         const mosca::ccd_config ccd_config,
         bool  perform_preoverscan);
/**
 * @brief    Define recipe parameters
 * @param    parameters     parameter list to fill
 * TODO: This was using fors_stack_define_parameters, but here we also offer
 * wmean by means of HDRL, therefore it is repeated. In the future, when
 * HDRL includes minmax,  fors_stack_define_parameters should be changed and
 * implemented via HDRL.
 */
void fors_bias_define_parameters(cpl_parameterlist *parameters)
{
    char *context = cpl_sprintf("fors.%s", fors_bias_name);
    
    cpl_parameter *p;
    char *full_name = NULL;
    const char *name;

    name = "stack_method";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_enum(full_name,
                               CPL_TYPE_STRING,
                               "Frames combination method",
                               context,
                               "minmax", 5,
                               "mean", "wmean", "median", "minmax", "ksigma");
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);


    /* minmax */
    name = "minrejection";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_INT,
                                "Number of lowest values to be rejected",
                                context,
                                1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);

    name = "maxrejection";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_INT,
                                "Number of highest values to be rejected",
                                context,
                                1);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);

    /* ksigma */
    name = "klow";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_DOUBLE,
                                "Low threshold in ksigma method",
                                context,
                                3.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);

    name = "khigh";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_DOUBLE,
                                "High threshold in ksigma method",
                                context,
                                3.0);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);

    name = "kiter";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_INT,
                                "Max number of iterations in ksigma method",
                                context,
                                999);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name);

    cpl_free((void *)context);

    return;
}

#undef cleanup
#define cleanup \
do { \
    cpl_frameset_delete(bias_frames); \
    fors_stack_method_delete(&sm); \
    cpl_free((void *)context); \
    fors_image_list_delete(&bias, fors_image_delete); \
    fors_image_delete(&master_bias); \
    fors_setting_delete(&setting); \
    cpl_propertylist_delete(qc); \
} while (0)
/**
 * @brief    Do the processing
 *
 * @param    frames         input frames
 * @param    parameters     recipe parameters
 *
 * @return   0 if everything is ok
 */

void fors_bias(cpl_frameset *frames, const cpl_parameterlist *parameters)
{
    /* Setting up groups */
    fors_dfs_set_groups(frames);

    /* Raw */
    cpl_frameset *bias_frames      = NULL;
    fors_image_list *bias    = NULL;

    /* Product */
    fors_image *master_bias = NULL;
    cpl_propertylist *qc = cpl_propertylist_new();

    /* Parameters */
    stack_method *sm    = NULL;

    /* Other */
    fors_setting *setting = NULL;
    char *context = cpl_sprintf("fors.%s", fors_bias_name);

    /* Get parameters */
    sm = fors_stack_method_new(parameters, context);
    assure( !cpl_error_get_code(), return, "Could not get stacking method");
    
    /* Find raw */
    bias_frames = fors_frameset_extract(frames, BIAS);
    assure( cpl_frameset_get_size(bias_frames) > 0, return, 
            "No %s provided", BIAS);

    /* Get instrument setting */
    setting = fors_setting_new(cpl_frameset_get_position(bias_frames, 0));
    cpl_propertylist * bias_header = 
            cpl_propertylist_load(cpl_frame_get_filename(
                    cpl_frameset_get_position(bias_frames, 0)), 0);
    fors::fiera_config bias_ccd_config(bias_header);
    cpl_propertylist_delete(bias_header);
    assure( !cpl_error_get_code(), return, "Could not get instrument setting" );

    /* Load bias */
    fors_image_list * bias_raw = 
            fors_image_load_list(bias_frames);
    fors_image * bias_first = fors_image_list_first(bias_raw);
    assure( !cpl_error_get_code(), return, "Could not load bias images");

    /* Compute RON */
    fors_bias_compute_ron(bias_raw, bias_ccd_config);
    
    /* Check if overscan is present in the raw images */
    bool perform_preoverscan = !fors_is_preoverscan_empty(bias_ccd_config);

    /* Create variances map */
    std::vector<double> overscan_levels;
    if(perform_preoverscan)
        overscan_levels = fors_get_bias_levels_from_overscan(bias_first,
                                                             bias_ccd_config);
    else
        overscan_levels = fors_get_bias_levels_from_mbias(bias_first,
                                                          bias_ccd_config);

    fors_image_variance_from_detmodel(bias_raw, bias_ccd_config, 
                                      overscan_levels);
    assure( !cpl_error_get_code(), return, "Cannot create variances map");
    
    /* Subtract overscan */
    if(perform_preoverscan)
        bias = fors_subtract_prescan(bias_raw, bias_ccd_config);
    else 
    {
        bias = fors_image_list_duplicate(bias_raw, fors_image_duplicate);
        //The rest of the recipe assumes that the images carry a bpm.
        fors_bpm_image_list_make_explicit(bias); 
    }
    assure( !cpl_error_get_code(), return, "Cannot subtract pre/overscan");

    /* Trimm pre/overscan */
    if(perform_preoverscan)
        fors_trimm_preoverscan(bias, bias_ccd_config);
    fors_image_list_delete(&bias_raw, fors_image_delete);
    assure( !cpl_error_get_code(), return, "Cannot trimm pre/overscan");

    /* Stack */
    master_bias = fors_bias_stack(bias, sm);
    assure( !cpl_error_get_code(), return, "Bias stacking failed");
    
    /* QC */
    write_qc(qc, setting,
             cpl_frameset_get_position(bias_frames, 0),
             bias, master_bias, sm, bias_ccd_config, perform_preoverscan);

    /* Save product */
    fors_dfs_save_image_err(frames, master_bias, MASTER_BIAS,
                        qc, NULL, parameters, fors_bias_name, 
                        cpl_frameset_get_position(bias_frames, 0));
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            MASTER_BIAS);
    
    cleanup;
    return;
}

#undef cleanup
#define cleanup \
do { \
} while (0)

fors_image *
fors_bias_stack(const fors_image_list *images, const stack_method *sm)
{
    fors_image *master_bias = NULL;
    cpl_image * contrib;

    assure( images != NULL, return master_bias, NULL );
    assure( fors_image_list_size(images) > 0, return master_bias, 
           "No images to collapse");
    
    cpl_msg_info(cpl_func, "Stacking bias images (method = %s)",
                 fors_stack_method_get_string(sm)); 

    hdrl_parameter * stackmethod_par = NULL;
    hdrl_imagelist * images_hdrl = NULL;
    hdrl_image * master_bias_hdrl = NULL;
    //Create parameters based on method
    switch (sm->method) {
    case stack_method::MEAN :
        stackmethod_par = hdrl_collapse_mean_parameter_create();
        break;
    case stack_method::WMEAN :
        stackmethod_par = hdrl_collapse_weighted_mean_parameter_create();
        break;
    case stack_method::MEDIAN : 
        stackmethod_par = hdrl_collapse_median_parameter_create();
        break;
    case stack_method::MINMAX :
        stackmethod_par = hdrl_collapse_minmax_parameter_create(
                sm->pars.minmax.min_reject, sm->pars.minmax.max_reject);
        break;
    case stack_method::KSIGMA : 
        stackmethod_par = hdrl_collapse_sigclip_parameter_create
          (sm->pars.ksigma.klow, sm->pars.ksigma.khigh, sm->pars.ksigma.kiter);
        break;
    default:
        assure( false, return NULL, "Unknown stack method '%s' (%d)",
                fors_stack_method_get_string(sm), sm->method);
        break;
    }
    //Do the stacking
    images_hdrl = fors_image_list_to_hdrl(images);
    hdrl_imagelist_collapse(images_hdrl, stackmethod_par ,&master_bias_hdrl, 
                            &contrib);
    assure( !cpl_error_get_code(), return NULL, "Collapsing of bias failed ");
    cpl_image_delete(contrib);
    master_bias = fors_image_from_hdrl(master_bias_hdrl);
    
    if(stackmethod_par != NULL)
        hdrl_parameter_destroy(stackmethod_par);
    if(images_hdrl != NULL)
        hdrl_imagelist_delete(images_hdrl);
    if(master_bias_hdrl != NULL)
        hdrl_image_delete(master_bias_hdrl);

    return master_bias;
}

void fors_bias_compute_ron
(fors_image_list * bias_raw, mosca::ccd_config& bias_ccd_config)
{
    for(size_t iport = 0; iport< bias_ccd_config.nports(); ++iport)
    {
        mosca::rect_region valid_region = 
                bias_ccd_config.validpix_region(iport).coord_0to1();
        std::vector<double> rons;
        const fors_image * target = fors_image_list_first_const(bias_raw);
        for(int ibias = 0; ibias< fors_image_list_size(bias_raw); ++ibias)
        {
            mosca::image bias(target->data, false);
            mosca::image bias_trimmed= bias.trim(valid_region.llx(),
                    valid_region.lly(), valid_region.urx(), valid_region.ury());
            float * data = bias_trimmed.get_data<float>();
            rons.push_back(mosca::robust_variance(data, 
                                                  data + bias_trimmed.npix()));
            target = fors_image_list_next_const(bias_raw);
        }
        double ron = std::sqrt(mosca::mean(rons.begin(), rons.end()));
        bias_ccd_config.set_computed_ron(iport, ron);
    }
}



#undef cleanup
#define cleanup \
do { \
    fors_image_delete(&first_bias_raw); \
    fors_image_delete(&image); \
} while (0)

static void
write_qc(cpl_propertylist *qc,
         const fors_setting *setting,
         const cpl_frame *first_bias,
         const fors_image_list *bias,
         const fors_image *master_bias,
         const stack_method *sm,
         const mosca::ccd_config ccd_config,
         bool  perform_preoverscan)
{
    const fors_image *first_raw  = fors_image_list_first_const(bias);
    const fors_image *second_raw = fors_image_list_next_const(bias);
    fors_image *image = NULL;

    (void)setting; //Avoid compiler warning

    fors_image * first_bias_raw = fors_image_load(first_bias);
    fors_header_write_double(qc,
                            fors_image_get_median(first_bias_raw, NULL),
                            "QC.BIAS.LEVEL",
                            "ADU",
                            "Bias level");
    double ron;
    double fpn;
    if (second_raw != NULL) {

        image = fors_image_duplicate(first_raw);
        fors_image_subtract(image, second_raw);

        ron = fors_image_get_stdev_robust(image, 50, NULL) / sqrt(2.0);

        fpn = fors_fixed_pattern_noise_bias(first_raw,
                                            second_raw,
                                            ron);
/*
        fpn = fors_fixed_pattern_noise(first_raw,
                                       1.0,
                                       ron);
*/
        assure( !cpl_error_get_code(), return, 
                "Could not compute fixed pattern noise" );
    }
    else {
        cpl_msg_warning(cpl_func,
                        "Only %d bias frame(s) provided, "
                        "cannot compute readout noise", 
                        fors_image_list_size(bias));
        ron = -1;
        fpn = -1;
    }

    fors_header_write_double(qc,
                            ron,
                            "QC.RON",
                            "ADU",
                            "Readout noise");

    fors_header_write_double(qc,
                            fpn,
                            "QC.BIAS.FPN",
                            "ADU",
                            "Bias fixed pattern noise");

    double structure = fors_image_get_stdev_robust(first_raw, 50, NULL);
    if (structure*structure >= ron*ron + fpn*fpn) {
        structure = sqrt(structure*structure - ron*ron - fpn*fpn);
    }
    else {
        cpl_msg_warning(cpl_func,
                        "Overall bias standard deviation (%f ADU) is less "
                        "than combined readout and fixed pattern noise "
                        "(%f ADU), setting structure to zero",
                        structure , sqrt(ron*ron + fpn*fpn));
        structure = 0;
    }
    
    
    fors_header_write_double(qc,
                            structure,
                            "QC.BIAS.STRUCT",
                            "ADU",
                            "Bias structure");

    /* Master bias QC */

    fors_header_write_double(qc,
                            fors_image_get_median(master_bias, NULL),
                            "QC.MBIAS.LEVEL",
                            "ADU",
                            "Master bias level");

    double ron_expect = -1;
    if (ron > 0) {

        int N = fors_image_list_size(bias);

        /*
          When median stacking and N >= 3, we need to
          take into account the fact that the median is more noisy than
          the mean.
        */

        if (sm->method == stack_method::MEDIAN) {
            ron_expect = fors_utils_median_corr(N) * ron / sqrt(N);
        }
        else {
            ron_expect = ron / sqrt(N);
        }
    }
    else cpl_msg_warning(cpl_func,
                         "Cannot compute expected master bias readout noise");
    
    fors_header_write_double(qc,
                            ron_expect,
                            "QC.MBIAS.RONEXP",
                            "ADU",
                            "Expected master bias readout noise");
    
    double mbias_stddev = -1;
    if (ron_expect > 0) {
        mbias_stddev =
            fors_image_get_stdev_robust(master_bias, 3*ron_expect, NULL);
    }
    else {
        mbias_stddev = -1;
    }
    
    fors_header_write_double(qc,
                            mbias_stddev,
                            "QC.MBIAS.NOISE",
                            "ADU",
                            "Master bias standard deviation");

    fors_header_write_double(qc,
                            mbias_stddev / ron_expect,
                            "QC.MBIAS.NRATIO",
                            NULL,
                            "Master bias observed/expected noise");
    
    double mbias_struct = fors_image_get_stdev(master_bias, NULL);

    if (mbias_struct * mbias_struct > mbias_stddev * mbias_stddev) {

        cpl_msg_debug(cpl_func, "Overall standard deviation is %f ADU",
                      mbias_struct);

        mbias_struct = sqrt(mbias_struct * mbias_struct - 
                mbias_stddev * mbias_stddev);
    }
    else {
        cpl_msg_warning(cpl_func,
                        "Master bias overall standard deviation (%f ADU) is "
                        "greater than master bias noise (%f ADU), "
                        "cannot compute master bias structure",
                        mbias_struct, mbias_stddev);
        mbias_struct = -1;
    }

    fors_header_write_double(qc,
                            mbias_struct,
                            "QC.MBIAS.STRUCT",
                            "ADU",
                            "Structure of master bias");
    
    //Save computed readout noise
    for(size_t iport = 0; iport< ccd_config.nports(); ++iport)
    {
        std::ostringstream key_stream;
        key_stream<<"QC.DET.OUT"<<iport+1<<".RON";
        fors_header_write_double(qc,
                                ccd_config.computed_ron(iport),
                                key_stream.str().c_str(),
                                "ADU",
                                "Computed readout noise per port");
    }

    
    cpl_propertylist_update_bool(qc, "ESO QC BIAS OVSC_SUB", 
                                     perform_preoverscan); 
    cpl_propertylist_set_comment(qc, "ESO QC BIAS OVSC_SUB", 
                                 "Whether overscan has been subtracted");
    assure( !cpl_error_get_code(), return, 
            "Could not update keyword QC BIAS OVSC_SUB");

    cleanup;
    return;
}
/**@}*/

