/* $Id: fors_img_science_impl.c,v 1.50 2013-09-10 19:16:03 cgarcia Exp $
 *
 * This file is part of the FORS Library
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

/*
 * $Author: cgarcia $
 * $Date: 2013-09-10 19:16:03 $
 * $Revision: 1.50 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <fors_img_science_impl.h>

#include <fors_extract.h>
#include <fors_tools.h>
#include <fors_setting.h>
#include <fors_data.h>
#include <fors_qc.h>
#include <fors_dfs.h>
#include <fors_utils.h>
#include "fiera_config.h"
#include "fors_overscan.h"
#include "fors_ccd_config.h"
#include "fors_detmodel.h"
#include <fors_subtract_bias.h>
#include "fors_bpm.h"
#include "fors_dfs_idp.h"

#include <cpl.h>
#include <math.h>
#include <stdbool.h>
#include <string.h>

/**
 * @addtogroup fors_img_science
 */

/**@{*/

const char *const fors_img_science_name = "fors_img_science";
const char *const fors_img_science_description_short = "Reduce scientific exposure";
const char *const fors_img_science_author = "Jonas M. Larsen";
const char *const fors_img_science_email = PACKAGE_BUGREPORT;
const char *const fors_img_science_description = 
"Input files:\n"
"  DO category:               Type:       Explanation:             			Number:\n"
"  SCIENCE_IMG                Raw         Science image               			 1\n"
"  MASTER_BIAS                FITS image  Master bias                 			 1\n"
"  MASTER_SKY_FLAT_IMG        FITS image  Master sky flat field       			 1\n"
//"  PHOT_COEFF_TABLE           FITS table  Observed extinction coefficients       0+\n"
//"  ALIGNED_PHOT           	  FITS table  Frame Zeropoint        				 0+\n"
"\n"
"Output files:\n"
"  DO category:               Data type:  Explanation:\n"
"  SCIENCE_REDUCED_IMG        FITS image  Reduced science image\n"
"  PHOT_BACKGROUND_SCI_IMG    FITS image  Reduced science image background\n"
"  SOURCES_SCI_IMG            FITS image  Unfiltered SExtractor output\n"
"  OBJECT_TABLE_SCI_IMG       FITS table  Extracted sources properties\n";


static double
get_image_quality(const fors_star_list *sources, double *image_quality_err,
                  double *stellarity,
                  double *ellipticity,
                  double *ellipticity_rms);

static fors_dfs_idp_converter *
generate_imaging_idp_converter(const cpl_frame * bias_frame,
		const cpl_frame * phot_frame, const fors_image * sci);

/**
 * @brief    Define recipe parameters
 * @param    parameters     parameter list to fill
 */

void fors_img_science_define_parameters(cpl_parameterlist *parameters)
{
    cpl_parameter *p;
    char *context = cpl_sprintf("fors.%s", fors_img_science_name);
    char *full_name = NULL;
    const char *name;

    /*  This parameter is not yet implemented
    name = "cr_remove";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_BOOL,
                                "Cosmic ray removal",
                                context,
                                false);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name); full_name = NULL;
    */

    name = "magsyserr";
    full_name = cpl_sprintf("%s.%s", context, name);
    p = cpl_parameter_new_value(full_name,
                                CPL_TYPE_DOUBLE,
                                "Systematic error in magnitude",
                                context,
                                0.01);
    cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, name);
    cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
    cpl_parameterlist_append(parameters, p);
    cpl_free((void *)full_name); full_name = NULL;

    fors_extract_define_parameters(parameters, context);
    
    cpl_free((void *)context);

    return;
}


#undef cleanup
#define cleanup \
do { \
    cpl_frameset_delete(sci_frame); \
    cpl_frameset_delete(master_bias_frame); \
    cpl_frameset_delete(master_flat_frame); \
    cpl_frameset_delete(phot_table); \
    fors_image_delete(&sci); \
    fors_image_delete(&master_bias); \
    fors_image_delete(&master_flat); \
    cpl_table_delete(phot); \
    cpl_table_delete(sources); \
    cpl_image_delete(background); \
    fors_extract_method_delete(&em); \
    fors_star_list_delete(&stars, fors_star_delete); \
    cpl_free((void *)context); \
    fors_setting_delete(&setting); \
    cpl_propertylist_delete(qc); \
    cpl_propertylist_delete(product_header); \
    cpl_propertylist_delete(header); \
} while (0)

/* %%% Removed from cleanup
    cpl_frameset_delete(phot_table); \
*/

/**
 * @brief    Do the processing
 *
 * @param    frames         set of frames
 * @param    parameters     parameters
 */
void fors_img_science(cpl_frameset *frames, const cpl_parameterlist *parameters)
{
    /* Raw */
    cpl_frameset *sci_frame      = NULL;
    fors_image *sci              = NULL;

    /* Calibration */
    cpl_frameset *master_bias_frame = NULL;
    fors_image *master_bias   = NULL; 

    cpl_frameset *master_flat_frame = NULL;
    fors_image *master_flat         = NULL; 


    cpl_frameset *phot_table        = NULL;

    /* Products */
    cpl_propertylist *qc = cpl_propertylist_new();
    cpl_propertylist *product_header = cpl_propertylist_new();
    cpl_propertylist *header = NULL;
    cpl_table *phot = NULL;
    fors_extract_sky_stats sky_stats;
    cpl_image *background = NULL;
    cpl_table *sources = NULL;

    /* Parameters */
    extract_method  *em = NULL;
    double           magsyserr;

    /* Other */
    char *context   = cpl_sprintf("fors.%s", fors_img_science_name);
    fors_star_list *stars = NULL;
    fors_setting *setting = NULL;
    double avg_airmass = 0.0;
    char *name;

    /* Get parameters */
    em = fors_extract_method_new(parameters, context);
    assure( !cpl_error_get_code(), return, 
            "Could not get extraction parameters" );

    cpl_msg_indent_more();
    name = cpl_sprintf("%s.%s", context, "magsyserr");
    magsyserr = dfs_get_parameter_double_const(parameters,
                                               name);
    cpl_free((void *)name); name = NULL;
    cpl_msg_indent_less();
    assure( !cpl_error_get_code(), return, NULL );
    assure( magsyserr >= 0, return, 
            "Input systematic error (magsyserr=%f) cannot be negative",
            magsyserr);
    
    /* Find raw */
    sci_frame = fors_frameset_extract(frames, SCIENCE_IMG);
    assure( cpl_frameset_get_size(sci_frame) == 1, return, 
            "Exactly 1 %s required. %"CPL_SIZE_FORMAT" found", 
            SCIENCE_IMG, cpl_frameset_get_size(sci_frame) );

    /* Find calibration */
    master_bias_frame = fors_frameset_extract(frames, MASTER_BIAS);
    assure( cpl_frameset_get_size(master_bias_frame) == 1, return, 
            "One %s required. %"CPL_SIZE_FORMAT" found", 
            MASTER_BIAS, cpl_frameset_get_size(master_bias_frame) );

    master_flat_frame = fors_frameset_extract(frames, MASTER_SKY_FLAT_IMG);
    assure( cpl_frameset_get_size(master_flat_frame) == 1, return, 
            "One %s required. %"CPL_SIZE_FORMAT" found", 
            MASTER_SKY_FLAT_IMG, cpl_frameset_get_size(master_flat_frame) );


    phot_table = fors_frameset_extract(frames, PHOT_COEFF_TABLE);
    cpl_frameset *zp_table = fors_frameset_extract(frames, ALIGNED_PHOT);

    if(cpl_frameset_get_size(phot_table) == 0 && cpl_frameset_get_size(zp_table) == 0){
    	//cpl_msg_warning(cpl_func, "Unable to extract %s or %s . Photometry IDP keywords"
		//    			" will not be produced.", PHOT_COEFF_TABLE, ALIGNED_PHOT);
    	cpl_frameset_delete(phot_table);
    	phot_table = NULL;
    	cpl_frameset_delete(zp_table);
    	zp_table = NULL;
    } else if(cpl_frameset_get_size(phot_table) != 0){
    	//    	cpl_msg_info(cpl_func, "Found %s. Photometry IDP keywords"
    	//   			" will be produced.", PHOT_COEFF_TABLE);
    	cpl_frameset_delete(zp_table);
    	zp_table = NULL;
    } else{
    	//cpl_msg_info(cpl_func, "%s was not found, but %s was found. Photometry IDP keywords"
    	//    	    			" will be produced.", PHOT_COEFF_TABLE, ALIGNED_PHOT);
    	cpl_frameset_delete(phot_table);
    	phot_table = NULL;
    	phot_table = zp_table;
    	zp_table = NULL;
    }

    /* Done finding frames */

    /* Get instrument setting */
    setting = fors_setting_new(cpl_frameset_get_position(sci_frame, 0));
    cpl_propertylist * sci_header = 
            cpl_propertylist_load(cpl_frame_get_filename(
                    cpl_frameset_get_position(sci_frame, 0)), 0);
    fors::fiera_config ccd_config(sci_header);    
    cpl_propertylist_delete(sci_header);
    assure( !cpl_error_get_code(), return, "Could not get instrument setting" );

    /* Update RON estimation from bias */
    cpl_propertylist * master_bias_header =
       cpl_propertylist_load(cpl_frame_get_filename(
               cpl_frameset_get_position(master_bias_frame, 0)), 0);
    fors::update_ccd_ron(ccd_config, master_bias_header);
    assure( !cpl_error_get_code(), return, "Could not get RON from master bias"
            " (missing QC DET OUT? RON keywords)");

    master_bias =
            fors_image_load(cpl_frameset_get_position(master_bias_frame, 0));
    assure( !cpl_error_get_code(), return, 
            "Could not load master bias");

    /* Load raw frames */
    fors_image * sci_raw = 
            fors_image_load(cpl_frameset_get_position(sci_frame, 0));
    assure( !cpl_error_get_code(), return, "Could not load science frames");

    /* Check that the overscan configuration is consistent */
    bool perform_preoverscan = !fors_is_preoverscan_empty(ccd_config);

    assure(perform_preoverscan == 
           fors_is_master_bias_preoverscan_corrected(master_bias_header),
           return, "Master bias overscan configuration doesn't match science");
    cpl_propertylist_delete(master_bias_header);
    
    /* Create variance */
    std::vector<double> overscan_levels; 
    if(perform_preoverscan)
        overscan_levels = fors_get_bias_levels_from_overscan(sci_raw, ccd_config);
    else
        overscan_levels = fors_get_bias_levels_from_mbias(master_bias, ccd_config);
    fors_image_variance_from_detmodel(sci_raw, ccd_config, overscan_levels);
    
    /*Subtract overscan */
    if(perform_preoverscan)
        sci = fors_subtract_prescan(sci_raw, ccd_config);
    else 
    {
        sci = fors_image_duplicate(sci_raw); 
        //The rest of the recipe assumes that the images carry a bpm.
        fors_bpm_image_make_explicit(sci); 
    }
    assure( !cpl_error_get_code(), return, "Could not subtract overscan");

    /* Trim pre/overscan */
    if(perform_preoverscan)
        fors_trimm_preoverscan(sci, ccd_config);
    fors_image_delete(&sci_raw);
    assure( !cpl_error_get_code(), return, "Could not trimm overscan");

    /* Subtract bias */
    fors_subtract_bias(sci, master_bias);
    assure( !cpl_error_get_code(), return, "Could not subtract master bias");
    fors_image_delete(&master_bias);

    /* Load master flat */
    master_flat =
            fors_image_load(cpl_frameset_get_position(master_flat_frame, 0));
    assure( !cpl_error_get_code(), return, "Could not load master flat");
    
    /* Divide by normalized flat */
    fors_image_divide_scalar(master_flat,
                             fors_image_get_median(master_flat, NULL), -1.0);
    assure( !cpl_error_get_code(), return, "Could not normalise flat");

    fors_image_divide(sci, master_flat);
    assure( !cpl_error_get_code(), return, "Could not divide by master flat");
    fors_image_delete(&master_flat);

    /* Extract sources */
    stars = fors_extract(sci, setting, em, magsyserr,
			 &sky_stats, &background, &sources);
    assure( !cpl_error_get_code(), return, "Could not extract objects");  

    /* QC */
    fors_qc_start_group(qc, fors_qc_dic_version, setting->instrument);
    
    fors_qc_write_group_heading(cpl_frameset_get_position(sci_frame, 0),
                                PHOTOMETRY_TABLE,
                                setting->instrument);
    assure( !cpl_error_get_code(), return, "Could not write %s QC parameters", 
            PHOTOMETRY_TABLE);


    double sky_mag;
    double sky_mag_rms;
    if (sky_stats.mean > 0) {
        sky_mag = -2.5*log(sky_stats.mean /
                           (setting->pixel_scale*setting->pixel_scale))/log(10);
    }
    else {
        cpl_msg_warning(cpl_func, 
                        "Average sky background is negative (%f ADU), "
                        "cannot compute magnitude, setting QC.SKYAVG to 99999.",
                        sky_stats.mean);
        sky_mag = 99999.;
    }
    fors_qc_write_qc_double(qc,
                            sky_mag,
                            "QC.SKYAVG",
                            "mag/arcsec^2",
                            "Mean of sky background",
                            setting->instrument);
    
    if (sky_stats.median > 0) {
        sky_mag = -2.5*log(sky_stats.median /
                           (setting->pixel_scale*setting->pixel_scale))/log(10);
        /* deltaM = -2.5*log10(e)*deltaF/F */
        sky_mag_rms = fabs(-2.5 * (1.0/log(10))*sky_stats.rms/sky_stats.median);
    }
    else {
        cpl_msg_warning(cpl_func, 
                        "Median sky background is negative (%f ADU), "
                        "cannot compute magnitude: setting both QC.SKYMED "
                        "and QC.SKYRMS to 99999.",
                        sky_mag);
        sky_mag = 99999.;
        sky_mag_rms = 99999.;
    }
    fors_qc_write_qc_double(qc,
                            sky_mag,
                            "QC.SKYMED",
                            "mag/arcsec^2",
                            "Median of sky background",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            sky_mag_rms,
                            "QC.SKYRMS",
                            "mag/arcsec^2",
                            "Standard deviation of sky background",
                            setting->instrument);

    double image_quality_error;
    double stellarity;
    double ellipticity, ellipticity_rms;
    double image_quality = get_image_quality(stars, 
                                             &image_quality_error,
                                             &stellarity,
                                             &ellipticity,
                                             &ellipticity_rms);

    if (image_quality > 0.) {
        image_quality *= TWOSQRT2LN2 * setting->pixel_scale;
        image_quality_error *= TWOSQRT2LN2 * setting->pixel_scale;
    }

    fors_qc_write_qc_double(qc,
                            image_quality,
                            "QC.IMGQU",
                            "arcsec",
                            "Image quality of scientific exposure",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            image_quality_error,
                            "QC.IMGQUERR",
                            "arcsec",
                            "Uncertainty of image quality",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            stellarity,
                            "QC.STELLAVG",
                            NULL,
                            "Mean stellarity index",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            ellipticity,
                            "QC.IMGQUELL",
                            NULL,
                            "Mean star ellipticity",
                            setting->instrument);

    fors_qc_write_qc_double(qc,
                            ellipticity_rms,
                            "QC.IMGQUELLERR",
                            NULL,
                            "Standard deviation of star ellipticities",
                            setting->instrument);

    fors_qc_end_group();

    /* Save SCIENCE_REDUCED, PHOT_BACKGROUND_SCI_IMG */

/* %%% */

    header = cpl_propertylist_load(
                        cpl_frame_get_filename(
                           cpl_frameset_get_position(sci_frame, 0)), 0);

    if (header == NULL) {
        cpl_msg_error(cpl_func, "Failed to load raw header");
        cleanup;
        return;
    }

    avg_airmass = fors_get_airmass(header);

    cpl_propertylist_update_double(qc, "AIRMASS", avg_airmass);
    cpl_propertylist_update_double(product_header, "AIRMASS", avg_airmass);

/* %%% */

    fors_dfs_add_wcs(qc, cpl_frameset_get_position(sci_frame, 0), setting);
    fors_dfs_add_exptime(qc, cpl_frameset_get_position(sci_frame, 0), 0.);
    fors_dfs_add_wcs(product_header, cpl_frameset_get_position(sci_frame, 0), 
                     setting);
    fors_dfs_add_exptime(product_header, cpl_frameset_get_position(sci_frame, 0), 0.);

    const cpl_frame * inherit_frame = cpl_frameset_get_position_const(sci_frame, 0);

    const cpl_frame * phot_frame = phot_table ?
    		cpl_frameset_get_position_const(phot_table, 0) : NULL;

    fors_dfs_idp_converter * converter =
    generate_imaging_idp_converter(cpl_frameset_get_frame(master_bias_frame, 0),
    		phot_frame, sci);

    fors_dfs_save_image_err_idp(frames, sci, SCIENCE_REDUCED_IMG,
                        qc, NULL, parameters, fors_img_science_name, 
						inherit_frame, NULL);

    fors_dfs_idp_converter_delete(converter);

    assure( !cpl_error_get_code(), return, "Saving %s failed",
            SCIENCE_REDUCED_IMG);

    fors_image_delete(&sci);
    
    dfs_save_image(frames, background, PHOT_BACKGROUND_SCI_IMG,
                   product_header, parameters, fors_img_science_name, 
                   setting->version);
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            PHOT_BACKGROUND_SCI_IMG);

    cpl_image_delete(background); background = NULL;
    
    /* Load filter coefficients */

/* %%%
    fors_phot_table_load(cpl_frameset_get_first(phot_table), setting,
                         NULL, NULL, 
			 &ext_coeff, &dext_coeff,
			 NULL, NULL);
    assure( !cpl_error_get_code(), return, "Could not load photometry table" );
*/

    /* Correct for atmospheric extinction */
/* %%%
    fors_star_ext_corr(stars, setting, ext_coeff, dext_coeff,
                       cpl_frameset_get_first(sci_frame));
    assure( !cpl_error_get_code(), return, 
            "Extinction correction failed");
*/

    /* Create, save FITS product */
    phot = fors_create_sources_table(stars);
    assure( !cpl_error_get_code(), return,
            "Failed to create extracted sources table");

    /*
     * Eliminate unused columns from photometry table
     */

    cpl_table_erase_column(phot, "INSTR_CMAG");
    cpl_table_erase_column(phot, "DINSTR_CMAG");
    cpl_table_erase_column(phot, "OBJECT");
    cpl_table_erase_column(phot, "MAG");
    cpl_table_erase_column(phot, "DMAG");
    cpl_table_erase_column(phot, "CAT_MAG");
    cpl_table_erase_column(phot, "DCAT_MAG");
    cpl_table_erase_column(phot, "COLOR");
    /* new columns since 4.4.10 */
    if (cpl_table_has_column(phot, "DCOLOR"))
        cpl_table_erase_column(phot, "DCOLOR");
    if (cpl_table_has_column(phot, "COV_CATM_COL"))
        cpl_table_erase_column(phot, "COV_CATM_COL");
    cpl_table_erase_column(phot, "USE_CAT");
    cpl_table_erase_column(phot, "SHIFT_X");
    cpl_table_erase_column(phot, "SHIFT_Y");
    cpl_table_erase_column(phot, "ZEROPOINT");
    cpl_table_erase_column(phot, "DZEROPOINT");
    cpl_table_erase_column(phot, "WEIGHT");

    fors_dfs_save_table(frames, sources, SOURCES_SCI,
                        NULL, parameters, fors_img_science_name, 
                        cpl_frameset_get_position(sci_frame, 0));
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            SOURCES_SCI);

    fors_dfs_save_table(frames, phot, PHOTOMETRY_TABLE,
                        NULL, parameters, fors_img_science_name, 
                        cpl_frameset_get_position(sci_frame, 0));
    assure( !cpl_error_get_code(), return, "Saving %s failed",
            PHOTOMETRY_TABLE);

    cleanup;
    return;
}

#undef cleanup
#define cleanup
/**
 * @brief    Determine if source is a star
 * @param    s             sources
 * @param    data          not used
 * @return   true iff the source is more star-like
 */
static bool
is_star(const fors_star *s, void *data)
{
    (void)data;
    assure( s != NULL, return false, NULL );

/*FIXME
  All stars for the moment... */

    return s->stellarity_index >= 0.7;
}

#undef cleanup
#define cleanup \
do { \
    fors_star_list_delete(&stars, fors_star_delete); \
} while(0)
/**
 * @brief    Compute image quality
 * @param    sources             extracted sources
 * @param    image_quality_err   (output) empirical scatter
 *                               (based on median absolute deviation)
 * @param    stellarity          (output) average stellarity of
 *                               sources above stellarity cutoff
 * @param    ellipticity         (output) average ellipticity of
 *                               sources above stellarity cutoff
 * @param    ellipticity_rms    (output) RMS ellipticity of
 *                               sources above stellarity cutoff
 * @param   median source extension (in the one sigma sense)
 */
static double
get_image_quality(const fors_star_list *sources, double *image_quality_err,
                  double *stellarity,
                  double *ellipticity,
                  double *ellipticity_rms)
{
    fors_star_list *stars = fors_star_list_extract(sources,
                                                   fors_star_duplicate,
                                                   is_star, NULL);

    double fwhm;
    if (fors_star_list_size(stars) >= 1) {
        *image_quality_err = fors_star_list_mad(stars, fors_star_extension, NULL) 
            * STDEV_PR_MAD;
        
        fwhm = fors_star_list_median(stars, fors_star_extension , NULL);
        
        *stellarity      = fors_star_list_mean(stars, fors_star_stellarity, NULL);
        *ellipticity     = fors_star_list_mean(stars, fors_star_ellipticity, NULL);
        *ellipticity_rms = fors_star_list_mad(stars, fors_star_ellipticity, NULL)
            * STDEV_PR_MAD;
    }
    else {
        cpl_msg_warning(cpl_func, "No stars found! Cannot compute image quality, "
                        "setting QC parameters to -1");

        /* -1 is not a valid value for any of these */
        *image_quality_err = -1;
        fwhm = -1;
        *stellarity = -1;
        *ellipticity = -1;
        *ellipticity_rms = -1;
    }

    cleanup;
    return fwhm;
}

static fors_dfs_idp_converter *
generate_imaging_idp_converter(const cpl_frame * bias_frame,
		const cpl_frame * phot_frame, const fors_image * sci){

	cpl_propertylist * bias_plist = cpl_propertylist_load(cpl_frame_get_filename(bias_frame), 0);

	if(bias_plist == NULL) return NULL;

	cpl_propertylist * phot_plist = NULL;
	if(phot_frame)
		phot_plist = cpl_propertylist_load(cpl_frame_get_filename(phot_frame), 0);

	fors_dfs_idp_converter * to_ret =
	fors_generate_imaging_idp_converter(bias_plist, phot_plist, sci);

	cpl_propertylist_delete(bias_plist);
	cpl_propertylist_delete(phot_plist);
	return to_ret;
}

/**@}*/
