/*
 * This file is part of the FORS Library
 * Copyright (C) 2002-2010 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <math.h>
#include <string>
#include <cstring>
#include <cmath>

#include "fors_img_idp.h"
#include "fors_dfs_idp.h"
#include "fors_utils.h"
#include "fors_dfs.h"
#include "fors_tools.h"


#undef cleanup
#define cleanup \
do { \
    fors_star_list_delete(&stars, fors_star_delete); \
} while(0)


#undef cleanup
#define cleanup
/**
 * @brief    Determine if source is eligible for
 * @param    s             sources
 * @param    data          not used
 * @return   true iff the source is more star-like and suitable for IDP generation
 */
static bool
is_usable_for_FWHM_IDP(const fors_star *s, void *data)
{
    (void)data;
    assure( s != NULL, return false, NULL );

    //Probable point sources
    const bool class_star = s->stellarity_index >= 0.8;

    //Negative total flux
    const bool mag_aper_neg = s->magnitude < 0.0;

    //FWHM positive
    const bool fwhm_pos = s->fwhm > 0.0;

    //No highly elongated sources
    const bool no_highly_enlong = s->semi_minor  != 0 &&
            (s->semi_major - s->semi_minor) / s->semi_minor  < 0.2;

    //avoid cosmic ray
    const bool no_cosmic = s->flux_max  != 0 && s->flux_aper / s->flux_max > 10.0;

    return class_star && mag_aper_neg && fwhm_pos && no_highly_enlong && no_cosmic;
}


#undef cleanup
#define cleanup

static
bool should_try_to_use_sources(const cpl_propertylist * header){
    if(!header) return false;

    const char * trak_status = "ESO TEL TRAK STATUS";
    const char * obs_name = "ESO OBS NAME";


    if(!cpl_propertylist_has(header, obs_name) ||
            !cpl_propertylist_has(header, trak_status)) return false;

    const char * trak = cpl_propertylist_get_string(header, trak_status);
    const char * not_mov = cpl_propertylist_get_string(header, obs_name);

    if(cpl_error_get_code() || trak == NULL || not_mov == NULL){
        cpl_error_reset();
        return false;
    }

    if(strcmp(trak, "NORMAL")) return false;

    const char * not_expected_mov = "MOV_";
    const cpl_size exp_length = strlen(not_expected_mov);
    if(strncmp(not_expected_mov, not_mov, exp_length) == 0)
        return false;

    return true;
}


#undef cleanup
#define cleanup

static
bool should_try_to_use_header(const cpl_propertylist * header){
    if(!header) return false;

    const char * trak_status = "ESO TEL TRAK STATUS";
    const char * obs_name = "ESO OBS NAME";


    if(cpl_propertylist_has(header, trak_status)) {
        const char * trak = cpl_propertylist_get_string(header, trak_status);
        if(trak != NULL && strcmp(trak, "DIFFERENTIAL")) return true;
    }

    if(cpl_propertylist_has(header, obs_name)) {
        const char * mov = cpl_propertylist_get_string(header, obs_name);
        const char * expected_mov = "MOV_";
        const cpl_size exp_length = strlen(expected_mov);
        if(mov != NULL && strncmp(expected_mov, mov, exp_length) == 0)
            return true;
    }

    cpl_error_reset();

    return false;
}

#undef cleanup
#define cleanup
static
bool try_to_use_header(const cpl_propertylist * header, double *ret){

    *ret = NAN;
    const char * def_key = "ESO TEL IA FWHMLINOBS";

    if(cpl_propertylist_has(header, def_key)) {
        *ret = cpl_propertylist_get_double(header, def_key);
        return true;
    }

    const char * fallback_key1 = "ESO TEL AMBI FWHM START";
    const char * fallback_key2 = "ESO TEL AIRM START";

    if(!cpl_propertylist_has(header, fallback_key1)) return false;
    if(!cpl_propertylist_has(header, fallback_key2)) return false;

    const double fwhm_start = cpl_propertylist_get_double(header, fallback_key1);
    const double amass_start = cpl_propertylist_get_double(header, fallback_key2);

    *ret = fwhm_start * pow(amass_start, 0.6);
    return true;
}

/**
 * @brief    Compute image quality
 * @param    sources             extracted sources
 * @param    fwhm_idp    (output)media fwhm on the selected sources
 * @param   TRUE in case of success
 */
bool fors_img_idp_get_image_psf_fwhm(const fors_star_list *sources,
        const fors_setting * settings,
        const cpl_propertylist * sci_header,
        double * fwhm_idp, double * fwhm_pix_idp){

    fors_star_list *stars = NULL;
    *fwhm_idp = NAN;
    if(sci_header == NULL ||
            sources == NULL || settings == NULL) return false;

    bool enough_sources = true;
    if(should_try_to_use_sources(sci_header))
    {
        stars = fors_star_list_extract(sources, fors_star_duplicate,
                is_usable_for_FWHM_IDP, NULL);

        const cpl_size n_selected = fors_star_list_size(stars);
        enough_sources = n_selected >= 5;
        const bool use_sources = stars != NULL
                && enough_sources//at least 5 sources
                && cpl_error_get_code() == CPL_ERROR_NONE;

        if (use_sources){
            const double med = fors_star_list_median(stars, fors_star_fwhm , NULL);
            *fwhm_idp = med * settings->pixel_scale; //fwhm is already multiplied by binx
            *fwhm_pix_idp = med;
            cleanup;
            fors_star_list_delete(&stars, fors_star_delete);
            return !std::isnan(*fwhm_idp);
        }
        fors_star_list_delete(&stars, fors_star_delete);
    }

    if(!enough_sources || should_try_to_use_header(sci_header)){
        const bool header_gen_success = try_to_use_header(sci_header, fwhm_idp);
        *fwhm_pix_idp = *fwhm_idp / settings->pixel_scale;
        cleanup;
        return header_gen_success && !std::isnan(*fwhm_idp);
    }

    cleanup;
    return false;
}

static fors_dfs_idp_converter *
generate_imaging_idp_converter(const cpl_frame * bias_frame,
        const fors_idp_zp_data * phot_data, const fors_image * sci,
        const double crder, const double ra, const double dec,
        const double skysqdeg, const double psf_fwhm,
        const double elliptic, const double ab_mag_lim, const double ab_mag_sat,
                const double exptime){

    cpl_propertylist * bias_plist = cpl_propertylist_load(cpl_frame_get_filename(bias_frame), 0);

    if(bias_plist == NULL) return NULL;

    fors_dfs_idp_converter * to_ret =
                    fors_generate_imaging_idp_converter(bias_plist, phot_data,
            sci, crder, ra, dec, skysqdeg,
            psf_fwhm, elliptic, ab_mag_lim, ab_mag_sat, exptime);

    cpl_propertylist_delete(bias_plist);
    return to_ret;
}


static const double default_extintion_error = 0.2;

static fors_idp_zp_data correct_zp(const double zp, const double zp_error, const double exptime,
        const double gain, const double extintion, const double extintion_err, const double amass){

    const double zp_out =  zp + 2.5 * log10(exptime) - 2.5 * log10(gain) - extintion * amass;

    cpl_msg_info(cpl_func, "Correcting zeropoint = %f with: exptime = %f. gain = %f, extinction = %f, airmass = %f",
            zp, exptime, gain, extintion, amass);

    cpl_msg_info(cpl_func, "Correcting zeropoint error = %f with: extinction_error = %f, airmass = %f",
            zp_error, extintion_err, amass);

    const double mult_error = amass * extintion_err;
    const double zp_err_out = sqrt(zp_error * zp_error + mult_error * mult_error);

    cpl_msg_info(cpl_func, "Corrected zeropoint = %f +/- %f",
            zp_out, zp_err_out);

    return (fors_idp_zp_data){zp_out, zp_err_out, extintion};
}

static bool fill_from_static_table(const cpl_frame * static_table, const cpl_frame * raw_science,
        const double exptime, const double gain, const double amass,
        fors_idp_zp_data * out_data, double * amag_vega){

    const char * raw_filt_name_key = "ESO INS FILT1 NAME";
    const char * filt_column_name = "FILTER";
    const char * zp_col = "ZEROPOINT";
    const char * zp_col_e = "ERROR";
    const char * ext_col_e = "EXTINCTION";
    const char * ab_vega_col_e = "ABVEGA";

    const char * fname = cpl_frame_get_filename(raw_science);
    cpl_propertylist * list = cpl_propertylist_load(fname, 0);

    if(!cpl_propertylist_has(list, raw_filt_name_key)){
        cpl_propertylist_delete(list);
        cpl_msg_warning(cpl_func, "The file %s does not contain a "
                "valid value of %s, aborting IDP generation", fname, raw_filt_name_key);
        return false;
    }

    const char * filt_name = cpl_propertylist_get_string(list, raw_filt_name_key);

    if(filt_name == NULL){
        cpl_propertylist_delete(list);
        cpl_msg_warning(cpl_func, "No valid filter name was provided, aborting IDP generation");
        return false;
    }

    cpl_table * tb = cpl_table_load(cpl_frame_get_filename(static_table), 1, CPL_FALSE);
    cpl_table_select_all(tb);

    std::string filt_name_regex = "^" + std::string(filt_name) + "$";
    cpl_table_and_selected_string(tb, filt_column_name, CPL_EQUAL_TO, filt_name_regex.c_str());

    const int n_selected = cpl_table_count_selected(tb);
    if(n_selected != 1){
        cpl_msg_warning(cpl_func, "The filter \"%s\" could not be found in the static table,"
                " aborting IDP generation because %i were selected", filt_name, n_selected);
        cpl_table_delete(tb);
        cpl_propertylist_delete(list);
        return false;
    }

    cpl_table * single_row = cpl_table_extract_selected(tb);
    cpl_table_delete(tb);

    const double ext = cpl_table_get_double(single_row, ext_col_e, 0, NULL);
    const double zp = cpl_table_get_double(single_row, zp_col, 0, NULL);

    const double zp_error = cpl_table_get_double(single_row, zp_col_e, 0, NULL);
    const double ext_err = default_extintion_error;

    *out_data = correct_zp(zp, zp_error, exptime, gain, ext, ext_err, amass);
    *amag_vega = cpl_table_get_double(single_row, ab_vega_col_e, 0, NULL);

    cpl_table_delete(single_row);
    cpl_propertylist_delete(list);

    const bool res = cpl_error_get_code() == CPL_ERROR_NONE;

    if(!res)
        cpl_error_reset();

    return res;
}

static cpl_boolean get_extinction_for_night(cpl_frameset * extinction_table,
        const double mjd_night, double * extinction, double * extinction_e){

    const cpl_frame * tb_f = cpl_frameset_get_position(extinction_table, 0);
    cpl_table * tb = cpl_table_load(cpl_frame_get_filename(tb_f), 1, 0);

    const char * night_col_tag = "MJD-NIGHT";
    const char * ext_col_tag = "EXT";
    const char * ext_error_col_tag = "RMS_NIGHT"; //This is not a typo.

    if(tb == NULL || cpl_error_get_code() ||
            !cpl_table_has_column(tb, night_col_tag) ||
            !cpl_table_has_column(tb, ext_col_tag)){
        cpl_table_delete(tb);
        return CPL_FALSE;
    }

    cpl_size best_match_idx = -1;
    double lowest_diff = INFINITY;
    double best_night = INFINITY;

    const cpl_size nrows = cpl_table_get_nrow(tb);
    for(cpl_size i = 0; i < nrows; ++i){

        const double night = (double)cpl_table_get_int(tb, night_col_tag, i, NULL);
        const double new_diff = fabs(night - mjd_night);
        if(new_diff < lowest_diff){
            lowest_diff = new_diff;
            best_match_idx = i;
            best_night = night;
        }

    }

    if(best_match_idx == -1){
        cpl_table_delete(tb);
        return CPL_FALSE;
    }

    cpl_msg_info(cpl_func, "The night %f was matched with the "
            "night %f in the extintion table", mjd_night, best_night);

    *extinction = cpl_table_get_double(tb, ext_col_tag, best_match_idx, NULL);

    *extinction_e = default_extintion_error; //If column not available, fall back to default
    if(cpl_table_has_column(tb, ext_error_col_tag)){
        *extinction_e = cpl_table_get_double(tb, ext_error_col_tag, best_match_idx, NULL);
    }

    if(*extinction_e <= 0)
                *extinction_e = default_extintion_error;


    cpl_table_delete(tb);
    return CPL_TRUE;
}



static bool fill_according_to_night(cpl_frameset * phot_tables,
        cpl_frameset * extinction_table, const cpl_frame * raw_science,
        const double exptime, const double gain, const double airmass,
        fors_idp_zp_data * out_data){

    const char * zp_key = "ESO QC INSTRUMENT ZEROPOINT";
    const char * zp_e_key = "ESO QC INSTRUMENT ZEROPOINT ERROR";

    double zp = 0;
    double zp_e = 0;

    const cpl_frame * phot_tb = cpl_frameset_get_position(phot_tables, 0);
    {
        const char * fname = cpl_frame_get_filename(phot_tb);
        cpl_propertylist * plist = cpl_propertylist_load(fname, 0);

        if(!cpl_propertylist_has(plist, zp_key) ||!cpl_propertylist_has(plist, zp_e_key)){
            cpl_propertylist_delete(plist);
            return false;
        }

        zp = cpl_propertylist_get_double(plist, zp_key);
        zp_e = cpl_propertylist_get_double(plist, zp_e_key);

        cpl_propertylist_delete(plist);
    }

    double extinction = 0;
    double extinction_e = 0;
    {
        const char * mjd_obs_tag = "MJD-OBS";
        cpl_propertylist * plist =
                cpl_propertylist_load(cpl_frame_get_filename(raw_science), 0);
        if(!cpl_propertylist_has(plist, mjd_obs_tag)){
            cpl_propertylist_delete(plist);
            return false;
        }
        const double mjd_obs = cpl_propertylist_get_double(plist, mjd_obs_tag);
        const double mjd_night = mjd_obs + 24.0e5;

        cpl_msg_info(cpl_func, "MJD-OBS = %f has been "
                "converted to MJD-NIGHT = %f", mjd_obs, mjd_night);

        if(!get_extinction_for_night(extinction_table, mjd_night, &extinction, &extinction_e)){
            cpl_propertylist_delete(plist);
            return false;
        }

        cpl_propertylist_delete(plist);
    }

    *out_data = correct_zp(zp, zp_e, exptime, gain, extinction, extinction_e, airmass);
    return true;
}


static bool extract_phot_table_for_IDP(const cpl_frameset * frames,
        const cpl_frame * raw_science,
        fors_idp_zp_data * out_data, double * amag_vega,
        const double exptime, const double gain,
        const double airmass){

    cpl_frameset * phot_table = fors_frameset_extract(frames, PHOT_COEFF_TABLE);
    cpl_frameset * extinction_table = fors_frameset_extract(frames, EXTINCTION_PER_NIGHT);
    cpl_frameset * static_table = fors_frameset_extract(frames, STATIC_PHOT_COEFF_TABLE);

    if(cpl_frameset_get_size(static_table) == 0){
        cpl_msg_warning(cpl_func, "Unable to extract %s. IDP products"
                                " will not be produced.", STATIC_PHOT_COEFF_TABLE);
        cpl_frameset_delete(phot_table);
        cpl_frameset_delete(static_table);
        cpl_frameset_delete(extinction_table);
        return false;
    }

    if(cpl_frameset_get_size(phot_table) == 1
            && cpl_frameset_get_size(extinction_table) == 0){
        cpl_msg_warning(cpl_func, "%s was provided, but %s  was not. IDP products"
                        " will not be produced.", PHOT_COEFF_TABLE,
                        EXTINCTION_PER_NIGHT);
        cpl_frameset_delete(phot_table);
        cpl_frameset_delete(static_table);
        cpl_frameset_delete(extinction_table);
        return false;
    }

    if(cpl_frameset_get_size(phot_table) == 0
            && cpl_frameset_get_size(extinction_table) == 1){
        cpl_msg_warning(cpl_func, "%s was provided, but %s  was not. IDP products"
                        " will not be produced.", EXTINCTION_PER_NIGHT,
                        PHOT_COEFF_TABLE);
        cpl_frameset_delete(phot_table);
        cpl_frameset_delete(static_table);
        cpl_frameset_delete(extinction_table);
        return false;
    }


    const cpl_frame * frame_static_table = cpl_frameset_get_position_const(static_table, 0);
    bool res =  fill_from_static_table(frame_static_table,
            raw_science, exptime, gain, airmass, out_data, amag_vega);

    if(cpl_frameset_get_size(phot_table) == 0
       && cpl_frameset_get_size(extinction_table) == 0){

        cpl_msg_info(cpl_func, "%s and %s not provided, "
                            "using the data in %s",
                            EXTINCTION_PER_NIGHT,
                            PHOT_COEFF_TABLE,
                            STATIC_PHOT_COEFF_TABLE);
        cpl_frameset_delete(phot_table);
        cpl_frameset_delete(static_table);
        cpl_frameset_delete(extinction_table);
        return res;
    }

    cpl_msg_info(cpl_func, "%s and %s provided, "
                        "using the data from them. %s "
                        "used only for AB Magnitude to VEGA conversions.",
                        EXTINCTION_PER_NIGHT,
                        PHOT_COEFF_TABLE,
                        STATIC_PHOT_COEFF_TABLE);

    res = fill_according_to_night(phot_table, extinction_table,
            raw_science, exptime, gain, airmass, out_data);

    cpl_frameset_delete(phot_table);
    cpl_frameset_delete(static_table);
    cpl_frameset_delete(extinction_table);

    return res;
}

static
cpl_image * get_weight_map(const fors_image * img, const double threshold){
    cpl_image * to_ret = cpl_image_duplicate(img->data);
    cpl_image_threshold(to_ret, threshold, threshold, 0, 1);
    return to_ret;
}

static double
get_crder(const cpl_frame * raw_science){

    const char * coll_key = "ESO INS COLL NAME";
    const char * fname = cpl_frame_get_filename(raw_science);
    cpl_propertylist * plist = cpl_propertylist_load(fname, 0);
    double to_ret = NAN;
    if(cpl_propertylist_has(plist, coll_key)){
        const char * coll_name = cpl_propertylist_get_string(plist, coll_key);

        if(!strcmp(coll_name, "COLL_SR"))
            to_ret = 1.0 / 3600.0;
        else if(!strcmp(coll_name, "COLL_HR"))
            to_ret = 2.7 / 3600.0;
        else
            cpl_msg_warning(cpl_func, "\"%s\" is an unrecognized "
                    "collimator, aborting IDP generation", coll_name);
    }

    cpl_propertylist_delete(plist);
    return to_ret;
}

static bool
get_data_from_raw_science(const cpl_frame * raw_frame,
        double * exptime, double * gain, double * airmass, double * pix_scale){
    cpl_propertylist * plist = cpl_propertylist_load(cpl_frame_get_filename(raw_frame), 0);

    const char * e_time_tag = "EXPTIME";
    const char * gain_tag = "ESO DET OUT1 CONAD";
    const char * pix_scale_tag = "ESO INS PIXSCALE";

    *exptime = cpl_propertylist_get_double(plist, e_time_tag);
    *airmass = fors_get_airmass(plist);
    *pix_scale = cpl_propertylist_get_double(plist, pix_scale_tag);
    *pix_scale /= 3600.0;
    *gain = cpl_propertylist_get_double(plist, gain_tag);
    bool success = cpl_error_get_code() == CPL_ERROR_NONE;

    if(!success)
        cpl_error_reset();

    cpl_propertylist_delete(plist);
    return success;
}

static bool convert_ra_dec(const cpl_propertylist * wcs_header,
        const cpl_image * weigth, double * ra, double * dec){

    cpl_wcs *wcs = cpl_wcs_new_from_propertylist(wcs_header);

    if(wcs == NULL || cpl_error_get_code()){
        cpl_error_reset();
        return false;
    }

    cpl_matrix * from_coord = cpl_matrix_new(1, 2);
    cpl_matrix * to_coord = NULL;
    cpl_array * status = NULL;

    bool success = true;
    const double center_weigth_x = cpl_image_get_centroid_x(weigth);
    const double center_weigth_y = cpl_image_get_centroid_y(weigth);

    cpl_matrix_set(from_coord, 0, 0, center_weigth_x);
    cpl_matrix_set(from_coord, 0, 1, center_weigth_y);

    if(cpl_wcs_convert(wcs, from_coord, &to_coord,
                 &status, CPL_WCS_PHYS2WORLD) != CPL_ERROR_NONE)
    {
        success = false;
        cpl_error_reset();
    }

    *ra  = cpl_matrix_get(to_coord, 0, 0);
    *dec = cpl_matrix_get(to_coord, 0, 1);

    cpl_msg_info(cpl_func, "Given the center of the weight map "
            "in pixels (%f, %f) the corresponding RA/DEC is (%f %f)",
            center_weigth_x, center_weigth_y, *ra, *dec);

    cpl_matrix_delete(from_coord);
    cpl_matrix_delete(to_coord);
    cpl_array_delete(status);
    cpl_wcs_delete(wcs);
    return success;
}

cpl_matrix * get_matrix_from_clm(const cpl_table * tb, const char * cl_name){

    const cpl_size sz = cpl_table_get_nrow(tb);
    cpl_matrix * m = cpl_matrix_new(1, sz);

    for(cpl_size i = 0; i < sz; ++i){
        const double d = cpl_table_get_float(tb, cl_name, i, NULL);
        cpl_matrix_set(m, 0, i, d);
    }
    return m;
}

cpl_vector * get_vector_from_clm(const cpl_table * tb, const char * cl_name){

    const cpl_size sz = cpl_table_get_nrow(tb);
    cpl_vector * v = cpl_vector_new(sz);

    for(cpl_size i = 0; i < sz; ++i){
        const double d = cpl_table_get_float(tb, cl_name, i, NULL);
        cpl_vector_set(v, i, d);
    }
    return v;
}


static bool get_flux_aper_in_saturation(const cpl_table * sources_selected,
        const char * x_clm, const char * y_clm, const double sky_med_level,
        const double overscan, double * flux_aper_in_sat){

    *flux_aper_in_sat = NAN;

    cpl_polynomial  * fit1d = cpl_polynomial_new(1);
    cpl_matrix * x_vals = get_matrix_from_clm(sources_selected, x_clm);
    cpl_vector * y_vals = get_vector_from_clm(sources_selected, y_clm);
    const cpl_size maxdeg1d  = 1;

    const cpl_error_code err = cpl_polynomial_fit(fit1d, x_vals, NULL, y_vals,  NULL,
            CPL_FALSE, NULL, &maxdeg1d);

    if(err || cpl_error_get_code()){
        cpl_msg_warning(cpl_func, "ABMASAT: Fit for calculation of "
                "ABMASAT failed, IDP generation aborted");
        cpl_error_reset();
    }

    if(!err){
        const double flux_max = pow(2.0, 16.0) - 1 - sky_med_level - overscan;
        *flux_aper_in_sat = cpl_polynomial_eval_1d(fit1d, flux_max, NULL);
        cpl_msg_debug(cpl_func, "ABMAGSAT: Using the fitted polynomial, given a "
                "flux max = %f (sky med level = %f), the corresponding flux "
                "aperture = %f", flux_max, sky_med_level, *flux_aper_in_sat);
    }

    cpl_matrix_delete(x_vals);
    cpl_vector_delete(y_vals);
    cpl_polynomial_delete(fit1d);

    return err == CPL_ERROR_NONE;
}


const char * ab_mag_sat_flags_clm = "FLAGS";
const char * ab_mag_sat_mag_aper_clm = "MAG_APER";
const char * ab_mag_sat_flx_err_aper_clm = "FLUXERR_APER";
const char * ab_mag_sat_flx_aper_clm = "FLUX_APER";
const char * ab_mag_sat_flx_max_clm = "FLUX_MAX";

static inline
void select_for_ab_mag_sat(cpl_table * sources,
        const cpl_boolean select_only_bright_sources){

    const cpl_size sz = cpl_table_get_nrow(sources);
    for(cpl_size i = 0; i < sz; ++i){

        const int flag = cpl_table_get_int(sources, ab_mag_sat_flags_clm, i, NULL);
        const double mag_aper = cpl_table_get_float(sources, ab_mag_sat_mag_aper_clm, i, NULL);
        const double flx_err_aper = cpl_table_get_float(sources, ab_mag_sat_flx_err_aper_clm, i, NULL);
        const double flx_aper = cpl_table_get_float(sources, ab_mag_sat_flx_aper_clm, i, NULL);
        const double flx_max = cpl_table_get_float(sources, ab_mag_sat_flx_max_clm, i, NULL);

        const bool is_bright =
                !select_only_bright_sources || flx_aper / flx_err_aper > 5.0 ;

        const bool select = flag == 0 && mag_aper < 0.0 &&
                is_bright && //we are interested in bright sources only
                flx_aper / flx_max > 10.0 && //cosmic events have usually flux restricted to 1-2 pixels
                flx_aper / flx_max < 100.0 &&  //filter out extended objects
                flx_max < 57000; //avoid non-linearity effects

        if(select)
            cpl_table_select_row(sources, i);
    }
}


static inline
cpl_table * select_sources_for_ab_mag_sat(const cpl_table * arg_sources){

    bool data_OK = cpl_table_has_column(arg_sources, ab_mag_sat_flags_clm) &&
            cpl_table_has_column(arg_sources, ab_mag_sat_mag_aper_clm) &&
            cpl_table_has_column(arg_sources, ab_mag_sat_flx_err_aper_clm) &&
            cpl_table_has_column(arg_sources, ab_mag_sat_flx_aper_clm) &&
            cpl_table_has_column(arg_sources, ab_mag_sat_flx_max_clm);

    if(!data_OK){
        cpl_msg_warning(cpl_func, "Sources table is missing one "
                "or more mandatory columns.");
        return NULL;
    }

    cpl_table * sources = cpl_table_duplicate(arg_sources);

    /*First try, use strict constraints*/
    cpl_table_unselect_all(sources);

    select_for_ab_mag_sat(sources, CPL_TRUE);

    int n_source_sel = cpl_table_count_selected(sources);
    if(n_source_sel >= 50) {
        cpl_table * sources_selected = cpl_table_extract_selected(sources);
        cpl_table_delete(sources);
        return sources_selected;
    }

    cpl_msg_warning(cpl_func, "ABMAGSAT: %i sources "
            "are not enough, relaxing selection criteria", n_source_sel);

    cpl_table_unselect_all(sources);
    select_for_ab_mag_sat(sources, CPL_FALSE);

    n_source_sel = cpl_table_count_selected(sources);

    if(n_source_sel < 2) {
        cpl_table_delete(sources);
        cpl_msg_warning(cpl_func, "ABMAGSAT: Not enough sources "
                "after relaxation of selection criteria, aborting");
        return NULL;
    }

    cpl_table * sources_selected = cpl_table_extract_selected(sources);
    cpl_table_delete(sources);
    return sources_selected;
}

static bool calc_ab_mag_sat(const cpl_table * arg_sources,
        const double sky_med_level, const double overscan,
        const double photo_zp, const double ab_vega, double * res){

    *res = NAN;

    cpl_table * sources_selected = select_sources_for_ab_mag_sat(arg_sources);

    if(sources_selected == NULL)
        return false;

    double flux_aper_in_saturation;
    bool success = get_flux_aper_in_saturation(sources_selected,
            ab_mag_sat_flx_max_clm, ab_mag_sat_flx_aper_clm, sky_med_level, overscan,
            &flux_aper_in_saturation);


    cpl_table_delete(sources_selected);

    if(success){
        const double instr_mag_sat = -2.5 * log10(flux_aper_in_saturation);
        cpl_msg_debug(cpl_func, "ABMAGSAT: Instrument mag sat = %f", instr_mag_sat);
        *res = instr_mag_sat + photo_zp + ab_vega;
    }


    cpl_msg_debug(cpl_func, "ABMAGSAT: Flux aperture "
            "in saturation = %f, zeropoint = %f, ab-vega = %f "
            "gives as ABMAGSAT = %f", flux_aper_in_saturation, photo_zp, ab_vega, *res);

    return success;
}

static double calc_ab_mag_lim(const double zp, const double ab_mag_minus_vega, const double sigma_sky_pix,
        const double fwhm_pix, const double aperture_correction){

    const double zp_ab = zp + ab_mag_minus_vega;

    cpl_msg_debug(cpl_func, "ABMAGLIM: zeropoint = %f, correction "
            "ab-vega = %f, zeropoint in AB used in the calculation = %f",
            zp, ab_mag_minus_vega, zp_ab);

    cpl_msg_debug(cpl_func, "ABMAGLIM: fwhm in pix = %f, "
                "sigma sky in pix = %f",
                fwhm_pix, sigma_sky_pix);

    //See formula in PIPE-8807
    const double abmaglim = zp_ab - 2.5 * log10(5.0 * sigma_sky_pix * sqrt(M_PI) * fwhm_pix / 2. / sqrt(log(4.))) - aperture_correction;

    return abmaglim;
}

static bool
get_sky_med_level(const cpl_frame * raw_frame, const double skymed, double * out){

    const char * pixscale_tag = "ESO INS PIXSCALE";

    cpl_propertylist * pl = cpl_propertylist_load(cpl_frame_get_filename(raw_frame), 0);

    const bool has_props = cpl_propertylist_has(pl, pixscale_tag) &&
            cpl_propertylist_get_type(pl, pixscale_tag) == CPL_TYPE_DOUBLE ;

    if(!has_props){
        cpl_propertylist_delete(pl);
        return false;
    }

    const double pixscale = cpl_propertylist_get_double(pl, pixscale_tag);

    cpl_propertylist_delete(pl);

    if(cpl_error_get_code()){
            cpl_error_reset();
            return false;
    }

    *out = pow(10.0, -0.4 * skymed) * pixscale * pixscale;

    return true;
}

void fors_img_idp_save(cpl_frameset * frames, const fors_image * sci,
        cpl_propertylist * qc, const cpl_parameterlist * parameters,
        const cpl_frame * raw_science_frame, const cpl_frame * master_bias,
        const fors_image * master_flat, const double threshold_weigth,
        fors_img_idp_tags fnames, fors_img_idp_quality qty){

    fors_idp_zp_data zp_data = {0, 0, 0};
    const double crder = get_crder(raw_science_frame);

    double exptime = 0.0;
    double gain = 0.0;
    double airmass = 0.0;
    double pix_scale = 0.0;

    if(!get_data_from_raw_science(raw_science_frame, &exptime, &gain, &airmass, &pix_scale)){
        cpl_msg_warning(cpl_func, "Failed extract either gain, exposure time or airmass, "
                "IDP products will not be saved.");
        return;
    }

    double abmag_minus_vega = 0.0;
    if(!extract_phot_table_for_IDP(frames, raw_science_frame, &zp_data, &abmag_minus_vega,
            exptime, gain, airmass) || std::isnan(crder))
    {
        cpl_msg_warning(cpl_func, "Failed to extract photometry data, "
                "IDP products will not be saved.");
        return;
    }

    cpl_image * weight_map = get_weight_map(master_flat, threshold_weigth);

    double ra = 0;
    double dec = 0;
    if(!convert_ra_dec(qc, weight_map, &ra, &dec)){
        cpl_msg_warning(cpl_func, "Failed to find the RA/DEC "
                "related to the center of the CCD, "
                "IDP products will not be saved.");
        cpl_image_delete(weight_map);
        return;
    }

    const double skysqdeg = pix_scale * pix_scale * cpl_image_get_flux(weight_map);
    const double aperture_correction = 0.0; //No aperture correction is performed
    const double ab_mag_lim = calc_ab_mag_lim(zp_data.zp, abmag_minus_vega, qty.sigma_sky,
            qty.psf_fwhm_pix, aperture_correction);
    double ab_mag_sat = 0.0;
    double sky_med_level = 0.0;

    if(!get_sky_med_level(raw_science_frame, qty.sky_mag, &sky_med_level)){
        cpl_msg_warning(cpl_func, "Failed to find sky med level, "
                "IDP products will not be saved.");
        cpl_image_delete(weight_map);
        return;
    }
    const bool success = calc_ab_mag_sat(qty.sources, sky_med_level, qty.pre_overscan_level,
            zp_data.zp, abmag_minus_vega, &ab_mag_sat);

    if(!success){
        cpl_msg_warning(cpl_func, "Calculation of ABMAGSAT failed, IDP generation aborted.");
        return;
    }

    cpl_propertylist * prod_catg = cpl_propertylist_new();
    cpl_propertylist_update_string(prod_catg, "PRODCATG", "ANCILLARY.MASK");
    cpl_propertylist_set_comment(prod_catg, "PRODCATG", "Data product category");

    fors_dfs_save_image(frames, weight_map, fnames.idp_weigths_name, prod_catg, parameters,
            fnames.recipe_name, raw_science_frame);

    cpl_image * sigma = cpl_image_power_create(sci->variance, 0.5);

    cpl_propertylist_update_string(prod_catg, "PRODCATG", "ANCILLARY.RMSMAP");

    fors_dfs_save_image(frames, sigma, fnames.idp_error_name, prod_catg, parameters,
            fnames.recipe_name, raw_science_frame);

    cpl_image_delete(sigma);
    cpl_propertylist_delete(prod_catg);

    fors_dfs_idp_converter * converter =
        generate_imaging_idp_converter(master_bias, &zp_data,
                sci, crder, ra, dec, skysqdeg, qty.psf_fwhm, qty.elliptic,
                ab_mag_lim, ab_mag_sat, exptime);


    fors_dfs_save_image_idp(frames, sci->data, fnames.idp_image_name,
                        qc, NULL, parameters, fnames.recipe_name,
                        raw_science_frame, converter);


    fors_dfs_idp_converter_delete(converter);
    cpl_image_delete(weight_map);
}
