#include "fors_spec_idp.h"
#include "global_distortion.h"
#include "wavelength_calibration.h"
#include "fors_dfs.h"
#include "irplib_sdp_spectrum.h"

#include <algorithm>
#include <string>
#include <cstring>
#include <cmath>
#include <sstream>

#define PI 3.14159265358979323846

const double angstrom_to_nm = 10.0;

static
double get_property_or_nan(const cpl_propertylist * l, const char * name){
    if(l == NULL) return NAN;

    double v = cpl_propertylist_get_double(l, name);

    if(cpl_error_get_code()){
        cpl_error_reset();
        return NAN;
    }

    return v;
}

static cpl_array *
extract_flux(const cpl_image * img, const cpl_size idx){

    const cpl_size sz_x = cpl_image_get_size_x(img);
    cpl_array * to_ret = cpl_array_new(sz_x, CPL_TYPE_DOUBLE);
    for(cpl_size x = 0; x < sz_x; ++x){
        int rej = 0;
        const double val = cpl_image_get(img, x + 1, idx + 1, &rej);
        if(!rej)
            cpl_array_set(to_ret, x, val);
        else
            cpl_array_set_invalid(to_ret, x);
    }

    return to_ret;
}

static cpl_array *
make_qual(const cpl_array * flx_data, const cpl_array * err_data) {
    const cpl_size flx_sz = cpl_array_get_size(flx_data);
    const cpl_size err_sz = cpl_array_get_size(err_data);
    if (flx_sz != err_sz) return NULL;

    cpl_array * qual_data = cpl_array_new(flx_sz, CPL_TYPE_INT);
    for (cpl_size x = 0; x < flx_sz; ++x) {
        int flx_rej = 0, err_rej = 0;
        const double flx_val = cpl_array_get(flx_data, x, &flx_rej);
        const double err_val = cpl_array_get(err_data, x, &err_rej);
        if (flx_rej || err_rej)  // 1 datum is invalid, -1 there was an error
            cpl_array_set_invalid(qual_data, x);
        else {
            // flx_val == err_val is a shorter check that gives *almost* the
            // same results as the following (unless there are legitimate cases
            // where the flux & err are the same but non-zero)
            const bool bad = flx_val == -1.0 || (flx_val == 0.0 && err_val == 0.0);
            cpl_array_set(qual_data, x, bad ? 1 : 0);
        }
    }
    return qual_data;
}

static const char * wave_units = "angstrom";
static const char * flx_units = "10**(-16)erg.cm**(-2).s**(-1).angstrom**(-1)";
static const char * flx_units_uncalib = "adu.s**(-1)";

static const char* type_qual = "spec:Data.FluxAxis.Accuracy.QualityStatus";
static const char* type_wave = "spec:Data.SpectralAxis.Value";
static const char* type_flx = "spec:Data.FluxAxis.Value";
static const char* type_flx_err = "spec:Data.FluxAxis.Accuracy.StatError";
static const char* type_flx_uncalib = "eso:Data.FluxAxis.Value";
static const char* type_flx_err_uncalib = "eso:Data.FluxAxis.Accuracy.StatError";

static const char * ucd_qual = "meta.code.qual;meta.main";
static const char * ucd_wave = "em.wl;obs.atmos";
static const char * ucd_flux = "phot.flux.density;em.wl;src.net;meta.main";
static const char * ucd_flux_err = "stat.error;phot.flux.density;meta.main";
static const char * ucd_flux_uncalib = "phot.flux.density;em.wl;src.net;stat.uncalib";
static const char * ucd_flux_err_uncalib = "stat.error;phot.flux.density;stat.uncalib";


static
irplib_sdp_spectrum * extract_1D_spectrum(const cpl_image * flx,
        const cpl_image * flx_error, const cpl_image * sky_bkg,
        const cpl_image * flx_reduced, const cpl_image * flx_error_reduced,
        const cpl_array * wlens, const cpl_size idx){

    irplib_sdp_spectrum * spectrum = irplib_sdp_spectrum_new();
    const cpl_size sz = cpl_image_get_size_x(flx);

    cpl_error_code fail = irplib_sdp_spectrum_set_nelem(spectrum, sz);
    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    cpl_array * flx_data = extract_flux(flx, idx);
    cpl_array * err_data = extract_flux(flx_error, idx);
    cpl_array * bkg_data = extract_flux(sky_bkg, idx);
    cpl_array * flx_red_data = extract_flux(flx_reduced, idx);
    cpl_array * flx_red_err_data = extract_flux(flx_error_reduced, idx);
    cpl_array * qual_data = make_qual(flx_data, err_data);

    fail = irplib_sdp_spectrum_add_column(spectrum, "WAVE", CPL_TYPE_DOUBLE, wave_units,
            NULL, type_wave, ucd_wave, wlens);

    if(!fail)
        fail = irplib_sdp_spectrum_replace_column_comment(spectrum, "WAVE",
                      "TUCD", "Air wavelength");

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    fail = irplib_sdp_spectrum_add_column(spectrum, "FLUX", CPL_TYPE_DOUBLE, flx_units,
            NULL, type_flx, ucd_flux, flx_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    fail = irplib_sdp_spectrum_add_column(spectrum, "ERR", CPL_TYPE_DOUBLE, flx_units,
            NULL, type_flx_err, ucd_flux_err, err_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }


    fail = irplib_sdp_spectrum_add_column(spectrum, "BGFLUX", CPL_TYPE_DOUBLE, flx_units_uncalib,
            NULL, type_flx_uncalib, ucd_flux_uncalib, bkg_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    fail = irplib_sdp_spectrum_add_column(spectrum, "FLUX_REDUCED", CPL_TYPE_DOUBLE, flx_units_uncalib,
            NULL, type_flx_uncalib, ucd_flux_uncalib, flx_red_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    fail = irplib_sdp_spectrum_add_column(spectrum, "ERR_REDUCED", CPL_TYPE_DOUBLE, flx_units_uncalib,
            NULL, type_flx_err_uncalib, ucd_flux_err_uncalib, flx_red_err_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    fail = irplib_sdp_spectrum_add_column(spectrum, "QUAL", CPL_TYPE_INT, "",
            NULL, type_qual, ucd_qual, qual_data);

    if(fail){
        irplib_sdp_spectrum_delete(spectrum);
        return NULL;
    }

    cpl_array_delete(flx_data);
    cpl_array_delete(err_data);
    cpl_array_delete(bkg_data);
    cpl_array_delete(flx_red_data);
    cpl_array_delete(flx_red_err_data);
    cpl_array_delete(qual_data);


    return spectrum;
}

static
char * get_pipe_id(const cpl_frame * frame)
{
    cpl_propertylist * plist = cpl_propertylist_load(cpl_frame_get_filename(frame), 0);
    if(!plist) return NULL;
    char * to_ret =  fors_dfs_pipeline_version(plist, NULL);
    cpl_propertylist_delete(plist);
    return to_ret;
}

static char *strlower(char *s)
{

  char *t = s;

  while (*t) {
    *t = tolower(*t);
    t++;
  }

  return s;

}

static void
propertylist_safe_append(cpl_propertylist * dest, const cpl_propertylist * source){

    for(cpl_size i = 0; i < cpl_propertylist_get_size(source); ++i){
        const cpl_property * p = cpl_propertylist_get_const(source, i);
        const char * source_name = cpl_property_get_name(p);
        if(cpl_propertylist_has(dest, source_name))
            continue;
        cpl_propertylist_append_property(dest, p);
    }
}

static
cpl_error_code save_spectrum_as_idp(
        fors_spec_idp_save_info& info,
        const irplib_sdp_spectrum * flx_points,
        const cpl_propertylist * extra_header,
        const cpl_propertylist * tabprops,
        const char * pipe_id,
        const cpl_size idx){

    char * fname = strlower(cpl_sprintf("%s_%lld.fits", info.idp_save_tag, idx));
    char * det_exp_num_idx = cpl_sprintf("%s_%lld", info.det_exp_num, idx);

    cpl_propertylist * pro = cpl_propertylist_new();
    cpl_propertylist_append_string(pro, "ESO PRO CATG", info.idp_save_tag);
    cpl_propertylist_append_string(pro, "ESO PRO DET_EXP_NUM", det_exp_num_idx);

    if(extra_header){
        propertylist_safe_append(pro, extra_header);
    }

    const cpl_error_code fail =
    irplib_dfs_save_spectrum(info.frameset, NULL, info.parlist,
            info.frameset, info.iherit_frame, flx_points, info.recipe_name,
            pro, tabprops, NULL, pipe_id, "PRO-1.15", fname);

    cpl_free(fname);
    cpl_free(det_exp_num_idx);
    cpl_propertylist_delete(pro);
    return fail;
}

static bool
has_string_property(const cpl_propertylist * plist, const char * name){

    return plist!= NULL && cpl_propertylist_has(plist, name) &&
                cpl_propertylist_get_type(plist, name) == CPL_TYPE_STRING;

}

static bool
has_int_property(const cpl_propertylist * plist, const char * name){

    return plist!= NULL && cpl_propertylist_has(plist, name) &&
                    cpl_propertylist_get_type(plist, name) == CPL_TYPE_INT;

}

static bool
add_obs(const cpl_propertylist * plist, irplib_sdp_spectrum * spec){

    const char * prog_id_tag = "ESO OBS PROG ID";
    const char * ob_id_tag = "ESO OBS ID";

    bool success = has_string_property(plist, prog_id_tag) && has_int_property(plist, ob_id_tag);

    if(!success) return false;

    const char * prog_id = cpl_propertylist_get_string(plist, prog_id_tag);
    const int obs_id = cpl_propertylist_get_int(plist, ob_id_tag);

    success = irplib_sdp_spectrum_set_progid(spec, prog_id) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_obid(spec, 1, obs_id) == CPL_ERROR_NONE;
    return success;
}


static bool
has_double_property(const cpl_propertylist * plist, const char * name){

    return plist!= NULL && cpl_propertylist_has(plist, name) &&
                cpl_propertylist_get_type(plist, name) == CPL_TYPE_DOUBLE;
}

static bool
add_exptimes_and_mjd(const cpl_propertylist * plist, irplib_sdp_spectrum * spec, const double exptime){

    bool success = true;
    const char * mjd = "MJD-OBS";

    const double mjd_obs = cpl_propertylist_get_double(plist, mjd);

    const double sec_in_day = 86400.0;
    const double mjd_end = mjd_obs + exptime / sec_in_day;

    success = success && irplib_sdp_spectrum_set_exptime(spec, exptime) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_texptime(spec, exptime) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_mjdobs(spec, mjd_obs) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_mjdend(spec, mjd_end) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_telapse(spec, (mjd_end - mjd_obs) * sec_in_day) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_tmid(spec, (mjd_end + mjd_obs) / 2.0) == CPL_ERROR_NONE;

    return success;
}

static bool
add_constants(irplib_sdp_spectrum * spectrum){

    bool success = irplib_sdp_spectrum_set_mepoch(spectrum, CPL_FALSE) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_ncombine(spectrum, 1) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_fluxcal(spectrum, "ABSOLUTE/UNCALIBRATED") == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_referenc(spectrum, "") == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_prodcatg(spectrum, "SCIENCE.SPECTRUM") == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_voclass(spectrum, "SPECTRUM V2.0") == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_vopub(spectrum, "ESO/SAF") == CPL_ERROR_NONE;

    return success;
}

static bool
add_pro(const cpl_propertylist * plist, irplib_sdp_spectrum * spec){
    
    const char * obs_tech_tag = "ESO PRO TECH";
    const char * pro_soft_tag = "ESO PRO REC1 PIPE ID";
    const char * pro_v_tag = "ESO PRO REC1 RAW1 NAME";

    bool success = has_string_property(plist, obs_tech_tag)
            && has_string_property(plist, pro_soft_tag)
            && has_string_property(plist, pro_v_tag);

    if(!success) return false;

    const char * obs_tech = cpl_propertylist_get_string(plist, obs_tech_tag);
    const char * pro_soft = cpl_propertylist_get_string(plist, pro_soft_tag);
    const char * pro_v = cpl_propertylist_get_string(plist, pro_v_tag);

    success = success &&irplib_sdp_spectrum_set_obstech(spec, obs_tech) == CPL_ERROR_NONE;
    success = success &&irplib_sdp_spectrum_set_procsoft(spec, pro_soft) == CPL_ERROR_NONE;
    success = success &&irplib_sdp_spectrum_set_prov(spec, 1, pro_v) == CPL_ERROR_NONE;
    return success;
}

static bool
add_qc_res(irplib_sdp_spectrum * spectrum,
        const cpl_propertylist * ids_coeff_pl,
        const double spec_sye){

    const char * resolution_tags = "ESO DRS RESOLUTION";
    const char * resolution_nwave_tags = "ESO DRS RESOLUTION NWAVE";
    const char * drs_accuracy_tags = "ESO DRS LAMRMS";

    bool success = has_int_property(ids_coeff_pl, resolution_nwave_tags)
            && has_double_property(ids_coeff_pl, resolution_tags)
            && has_double_property(ids_coeff_pl, drs_accuracy_tags);

    if(!success) return false;

    const int lamlin = cpl_propertylist_get_int(ids_coeff_pl, resolution_nwave_tags);
    const double spec_res = cpl_propertylist_get_double(ids_coeff_pl, resolution_tags);
    const double lamrms = cpl_propertylist_get_double(ids_coeff_pl, drs_accuracy_tags);

    success = irplib_sdp_spectrum_set_lamnlin(spectrum, lamlin) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_specres(spectrum, spec_res) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_lamrms(spectrum, lamrms) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_specerr(spectrum, lamrms / sqrt(lamlin)) == CPL_ERROR_NONE;
    // Don't fail if spec_sye isn't valid - as per PIPE-9233 it's optional
    if (!std::isnan(spec_sye)) {
        irplib_sdp_spectrum_set_specsye(spectrum, spec_sye);
    }
    return success;
}

static bool
add_grism_data_elem(irplib_sdp_spectrum * spectrum, const cpl_propertylist * pl){

    const char * disp_elem = "ESO INS GRIS1 NAME";
    const char * spec_val_elem = "ESO INS GRIS1 WLEN";

    if(!has_string_property(pl, disp_elem)) return false;

    if(!has_double_property(pl, spec_val_elem)) return false;

    const char * disp_el = cpl_propertylist_get_string(pl, disp_elem);

    if(irplib_sdp_spectrum_set_dispelem(spectrum, disp_el) != CPL_ERROR_NONE)
        return false;

    const double spec_val = cpl_propertylist_get_double(pl, spec_val_elem);

    return irplib_sdp_spectrum_set_specval(spectrum, spec_val) == CPL_ERROR_NONE;
}

static double get_min_valid_wlen(const irplib_sdp_spectrum * spectrum){

    if(spectrum == NULL) return NAN;

    int hole;
    const cpl_array * wlens_angstrom = irplib_sdp_spectrum_get_column_data(spectrum, "WAVE");
    const cpl_array * qual = irplib_sdp_spectrum_get_column_data(spectrum, "QUAL");

    if(!qual || !wlens_angstrom) return NAN;

    const cpl_size sz = cpl_array_get_size(qual);
    for(cpl_size i = 0; i < sz; ++i)
    {
        int qualrej = 0;
        const double qualval = cpl_array_get(qual, i, &qualrej);
        if(!qualrej && !qualval)
            return cpl_array_get(wlens_angstrom, i, &hole);
    }

    return cpl_array_get(wlens_angstrom, 0, &hole);
}

static double get_max_valid_wlen(const irplib_sdp_spectrum * spectrum){

    if(spectrum == NULL) return NAN;

    int hole;
    const cpl_array * wlens_angstrom = irplib_sdp_spectrum_get_column_data(spectrum, "WAVE");
    const cpl_array * qual = irplib_sdp_spectrum_get_column_data(spectrum, "QUAL");

    if(!qual || !wlens_angstrom) return NAN;

    const cpl_size last_el = cpl_array_get_size(qual) - 1;
    for(cpl_size i = last_el; i >= 0; --i)
    {
        int qualrej = 0;
        const double qualval = cpl_array_get(qual, i, &qualrej);
        if(!qualrej && !qualval)
            return cpl_array_get(wlens_angstrom, i, &hole);
    }

    return cpl_array_get(wlens_angstrom, last_el, &hole);
}

static bool add_dimensions_elem(irplib_sdp_spectrum * spectrum,
                                const cpl_array * wlens){

    const double wmin_a = get_min_valid_wlen(spectrum);
    const double wmax_a = get_max_valid_wlen(spectrum);

    const double wmin_nm = wmin_a / angstrom_to_nm;
    const double wmax_nm = wmax_a / angstrom_to_nm;

    int null_val_min = 0, null_val_max = 0;
    const double tdmin = cpl_array_get_double(wlens, 0, &null_val_min);
    const double tdmax = cpl_array_get_double(wlens, cpl_array_get_size(wlens)-1, &null_val_max);

    if(std::isnan(wmin_nm) || std::isnan(wmax_nm) ||
       null_val_min || null_val_max || cpl_error_get_code()){
        cpl_error_reset();
        return false;
    }

    bool ret = irplib_sdp_spectrum_set_wavelmax(spectrum, wmax_nm) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_wavelmin(spectrum, wmin_nm) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_specbw(spectrum, wmax_nm - wmin_nm) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_specval(spectrum, (wmax_nm + wmin_nm) / 2.0) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_tdmin(spectrum, tdmin) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_tdmax(spectrum, tdmax) == CPL_ERROR_NONE;
    return ret;
}

static bool add_spectrum_specific_metadata(const cpl_propertylist * pl,
        irplib_sdp_spectrum * spectrum, const cpl_array * wlens){

    bool ret = irplib_sdp_spectrum_set_specsys(spectrum, "TOPOCENT") == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_contnorm(spectrum, CPL_FALSE) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_totflux(spectrum, CPL_FALSE) == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_fluxcal(spectrum, "ABSOLUTE") == CPL_ERROR_NONE;
    ret = ret && irplib_sdp_spectrum_set_fluxerr(spectrum, -2.0) == CPL_ERROR_NONE;
    ret = ret && add_grism_data_elem(spectrum, pl);
    ret = ret && add_dimensions_elem(spectrum, wlens);
    return ret;
}

static double get_median(std::vector<double>& v){
    size_t n = v.size() / 2;
    std::nth_element(v.begin(), v.begin()+n, v.end());
    return v[n];
}

static std::vector<double> get_moses(const cpl_propertylist * sci_reduced_header, const int start){
    std::vector<double> mos;
    int i = start;
    while (true) {

            std::stringstream ss;
            ss<<"ESO INS MOS"<<i<<" WID";
            if(!cpl_propertylist_has(sci_reduced_header, ss.str().c_str()))
                break;

            const double val = cpl_propertylist_get_double(sci_reduced_header, ss.str().c_str());

            /*Only actually used slitlets have the following keyword set*/
            ss.str("");
            ss<<"ESO INS MOS"<<i<<" LEN";
            i++;
            if(!cpl_propertylist_has(sci_reduced_header, ss.str().c_str()))
                continue;

            mos.push_back(val);
    }
    return mos;
}

static bool set_aper_for_lss(const cpl_propertylist * sci_reduced_header, irplib_sdp_spectrum * spectrum){
    const char * lss_tag = "ESO INS SLIT WID";

    if(cpl_propertylist_has(sci_reduced_header, lss_tag)){
        double val = cpl_propertylist_get_double(sci_reduced_header, lss_tag);
        irplib_sdp_spectrum_set_aperture(spectrum, val/3600.0);
        return true;
    }

    return false;
}

static bool set_aper_for_mos(const cpl_propertylist * sci_reduced_header, irplib_sdp_spectrum * spectrum, const int start){

    std::vector<double> mos = get_moses(sci_reduced_header, start);

    if(mos.empty())
        return false;

    /*If slit identification is disabled we cannot reconstruct which slit an
     * object belongs to, use the median, usually they are all the same*/
    double median_wid = get_median(mos);

    if(std::isnan(median_wid)) return false;

    irplib_sdp_spectrum_set_aperture(spectrum, median_wid/3600.0);

    return true;
}

static bool
set_mos_wid(const cpl_propertylist * sci_reduced_header, irplib_sdp_spectrum * spectrum){

    const char * mode_tag = "ESO INS MODE";
    if(!has_string_property(sci_reduced_header, mode_tag))
        return false;

    const char * mode = cpl_propertylist_get_string(sci_reduced_header, mode_tag);

    if(!strcmp(mode, "LSS")){

        return set_aper_for_lss(sci_reduced_header, spectrum);

    }


    if(!strcmp(mode, "MOS")){
        return set_aper_for_mos(sci_reduced_header, spectrum, 1);
    }

    if(!strcmp(mode, "MXU")){
            /*In MXU slitlets from 101 to 106 are no used for science observations*/
            return set_aper_for_mos(sci_reduced_header, spectrum, 107);
    }

    return false;
}

static bool
add_bin(const cpl_propertylist * sci_reduced_header, irplib_sdp_spectrum * spectrum){

    const char * cd_tag = "CD1_1";
    bool success = has_double_property(sci_reduced_header, cd_tag);

    if(!success) return false;

    double cd = cpl_propertylist_get_double(sci_reduced_header, cd_tag) / angstrom_to_nm;
    return irplib_sdp_spectrum_set_specbin(spectrum, cd) == CPL_ERROR_NONE;
}


static double calc_snr(const irplib_sdp_spectrum * spectrum){
    const cpl_array * flx = irplib_sdp_spectrum_get_column_data(spectrum, "FLUX");
    const cpl_array * err = irplib_sdp_spectrum_get_column_data(spectrum, "ERR");

    cpl_array * div = cpl_array_duplicate(flx);
    cpl_array_divide(div, err);

    int valid;
    for(int i=0; i<cpl_array_get_size(flx); ++i)
        if(cpl_array_get_double(flx, i, &valid)==-1 &&
           cpl_array_get_double(err, i, &valid)==-1)
            cpl_array_set_invalid(div, i);

    const double ret = cpl_array_get_median(div);
    cpl_array_delete(div);

    if(cpl_error_get_code()) {
        cpl_error_reset();
        return NAN;
    }

    return ret;
}

static bool add_snr(irplib_sdp_spectrum * spectrum){

    const double snr = calc_snr(spectrum);
    if(std::isnan(snr)) return false;
    return irplib_sdp_spectrum_set_snr(spectrum, snr) == CPL_ERROR_NONE;
}

static bool add_object(const cpl_propertylist * sci_reduced_header, 
                       irplib_sdp_spectrum * spectrum,
                       const char* det_exp_num,
                       cpl_size idx_spectrum){

    const char * ob_tag = "OBJECT";
    bool success = has_string_property(sci_reduced_header, ob_tag);

    if(!success) return false;

    const char * ob = cpl_propertylist_get_string(sci_reduced_header, ob_tag);
    std::stringstream oss;
    oss << ob << "_" << det_exp_num << "_" << idx_spectrum+1;
    success = success && irplib_sdp_spectrum_set_object(spectrum, ob) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_title(spectrum, oss.str().c_str()) == CPL_ERROR_NONE;
    return success;
}

static
bool add_spectrum_metadata(irplib_sdp_spectrum * spectrum ,
        const cpl_array * wlens,
        const cpl_propertylist * sci_reduced_header,
        const cpl_propertylist * ids_header,
        const double ra, const double dec,
        const double ron, const double spec_sye,
        const double exptime, const char* det_exp_num,
        cpl_size idx_spectrum){

    if(sci_reduced_header == NULL || spectrum == NULL) return false;

    bool success = irplib_sdp_spectrum_set_ra(spectrum, ra) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_dec(spectrum, dec) == CPL_ERROR_NONE;
    success = success && add_exptimes_and_mjd(sci_reduced_header, spectrum, exptime);
    success = success && add_obs(sci_reduced_header, spectrum);
    success = success && add_constants(spectrum);
    success = success && add_pro(sci_reduced_header, spectrum);
    success = success && irplib_sdp_spectrum_set_detron(spectrum, ron) == CPL_ERROR_NONE;
    success = success && irplib_sdp_spectrum_set_effron(spectrum, ron) == CPL_ERROR_NONE;
    success = success && add_spectrum_specific_metadata(sci_reduced_header, spectrum, wlens);
    success = success && set_mos_wid(sci_reduced_header, spectrum);
    success = success && add_qc_res(spectrum, ids_header, spec_sye);
    success = success && add_bin(sci_reduced_header, spectrum);
    success = success && add_snr(spectrum);
    success = success && add_object(sci_reduced_header, spectrum, det_exp_num, idx_spectrum);

    return success;
}

typedef std::pair<double, double> ra_dec;
static std::vector<ra_dec> get_ra_dec(const cpl_table * objects)
{
    cpl_size maxobjects = 1;
    std::stringstream name_stream;
    name_stream<<"object_"<<maxobjects;

    while (cpl_table_has_column(objects, name_stream.str().c_str())) {
        maxobjects++;
        name_stream.str(std::string());
        name_stream<<"object_"<<maxobjects;
    }

    /*
     * Count objects to extract
     */

    cpl_size nslits = cpl_table_get_nrow(objects);

    cpl_size obj_count = 0;
    for (cpl_size i = 0; i < nslits; i++) {
        for (cpl_size j = 1; j < maxobjects; j++) {
            std::stringstream name_stream_obj;
            name_stream_obj << "object_" << j;
            if (cpl_table_is_valid(objects, name_stream_obj.str().c_str(), i)) {
                ++obj_count;
            }
        }
    }

    /* Allocate the vector to return */
    std::vector<ra_dec> to_ret;
    to_ret.resize(obj_count);
    for (cpl_size i = 0; i < nslits; i++) {
        for (cpl_size j = 1; j < maxobjects; j++) {
            std::stringstream name_stream_obj;
            name_stream_obj << "object_" << j;
            if (cpl_table_is_valid(objects, name_stream_obj.str().c_str(), i)) {

                std::stringstream ra_stream; ra_stream << "ra_"<<j;
                std::stringstream dec_stream; dec_stream << "dec_"<<j;
                std::stringstream row_stream; row_stream << "row_"<<j;

                const bool is_ra_valid = cpl_table_is_valid(objects, ra_stream.str().c_str(), i);
                const bool is_dec_valid = cpl_table_is_valid(objects, dec_stream.str().c_str(), i);
                const bool is_row_valid = cpl_table_is_valid(objects, row_stream.str().c_str(), i);

                /* Find the row that this radec applies to */
                if (!is_row_valid) {
                    cpl_msg_warning(cpl_func, "For slits %lld and object %lld row could "
                                              "not be determined, aborting IDP generation", i, j);
                    to_ret.clear();
                    return to_ret;
                }

                const long row = cpl_table_get_int(objects, row_stream.str().c_str(), i, NULL);
                if ((row < 0) || ((long unsigned)row >= to_ret.size())) {
                    cpl_msg_warning(cpl_func, "For slits %lld and object %lld row %ld "
                                              "is invalid, aborting IDP generation", i, j, row);
                    to_ret.clear();
                    return to_ret;
                }
                
                /* Check that the radec is valid */
                if (is_ra_valid && is_dec_valid) {
                    const double ra = cpl_table_get_double(objects, ra_stream.str().c_str(), i, NULL);
                    const double dec = cpl_table_get_double(objects, dec_stream.str().c_str(), i, NULL);
                    /* Store the radec against the row number */
		    to_ret[row] = ra_dec(ra, dec);
                } else {
                    cpl_msg_warning(cpl_func, "For slits %lld and object %lld RA/DEC could "
                                              "not be determined, aborting IDP generation", i, j);
                    to_ret.clear();
                    return to_ret;
                }
            }
        }
    }

    return to_ret;
}

static double
get_double_from_int_or_default(const cpl_propertylist * p, const char * name, const double default_val){

    if(!has_int_property(p, name))
        return default_val;

    return (double)cpl_propertylist_get_int(p, name);
}

static double
get_double_or_default(const cpl_propertylist * p, const char * name, const double default_val){

    if(!has_double_property(p, name))
        return default_val;

    return cpl_propertylist_get_double(p, name);
}

/* makes use of crder, so must run after it has been calculated */
static
cpl_error_code add_radec_err(const cpl_propertylist * source, cpl_propertylist * dest) {
    // VLT positioning rms error [3 arcsec] projected equally along RA and DEC
    // rms positioning error: 3 arcsec
    const double err_vlt_ra = 3. / 3600. / sqrt(2.);
    const double err_vlt_dec = 3. / 3600. / sqrt(2.);

    const double posang = get_property_or_nan(source, "ESO ADA POSANG");
    const double crder = get_property_or_nan(dest, "ESO DRS CRDER");
    if (std::isnan(posang) || std::isnan(crder)) return CPL_ERROR_DATA_NOT_FOUND;

    const double cos_posang = cos(posang * PI / 180.0);
    const double sin_posang = sin(posang * PI / 180.0);

    // the error due to the wavelength calibration projected along the RA and
    // DEC axes
    const double err_disp_ra = std::abs(crder / 3600. * cos_posang);
    const double err_disp_dec = std::abs(crder / 3600. * sin_posang);

    irplib_sdp_spectrum * spectrum = irplib_sdp_spectrum_new();
    if (!spectrum || !set_mos_wid(source, spectrum)) {
        if (spectrum) irplib_sdp_spectrum_delete(spectrum);
        return CPL_ERROR_INCOMPATIBLE_INPUT;
    }

    double med_slit_width = irplib_sdp_spectrum_get_aperture(spectrum);
    irplib_sdp_spectrum_delete(spectrum);
    if (std::isnan(med_slit_width)) return CPL_ERROR_DATA_NOT_FOUND;

    med_slit_width *= 3600.0;  // convert to arcsec

    // the possible offset of a target from the center of the slit along the
    // dispersion axis (0.3 x slit width [arcsec]), projected along the RA and
    // DEC axes
    const double err_slit_ra = abs((med_slit_width / 3.) / 3600. * cos_posang);
    const double err_slit_dec = abs((med_slit_width / 3.) / 3600. * sin_posang);

    const double ra_err = sqrt(pow(err_vlt_ra, 2.) + pow(err_disp_ra, 2.) +
            pow(err_slit_ra, 2.));
    const double dec_err = sqrt(pow(err_vlt_dec, 2.) + pow(err_disp_dec, 2.) +
            pow(err_slit_dec, 2.));

    cpl_error_code rv = CPL_ERROR_NONE;
    // here (below), short-circuiting stops evaluation after the 1st failure
    if ((rv = cpl_propertylist_update_double(dest, "RA_ERR", ra_err    )) ||
        (rv = cpl_propertylist_set_comment(dest, "RA_ERR", "Error on "
            "spectroscopic target position [deg]"                      )) ||
        (rv = cpl_propertylist_update_double(dest, "DEC_ERR", dec_err  )) ||
        (rv = cpl_propertylist_set_comment(dest, "DEC_ERR", "Error on "
            "spectroscopic target position [deg]"                      ))
    ) return rv;

    return CPL_ERROR_NONE;
}

static double calc_crder(const cpl_propertylist * sci_reduced_header,
        const cpl_propertylist * ids_coeff_pl){

    const char * drs_accuracy_tags = "ESO DRS LAMRMS";
    const char * cd_tag = "CD1_1";
    const char * pix_size_tags = "ESO DET CHIP1 PSZX";
    const char * bin_size_tags = "ESO DET WIN1 BINX";

    const double lamrms = get_double_or_default(ids_coeff_pl, drs_accuracy_tags, NAN);
    const double cd = get_double_or_default(sci_reduced_header, cd_tag, NAN) / angstrom_to_nm;
    const double pix_size = get_double_or_default(sci_reduced_header, pix_size_tags, NAN) / angstrom_to_nm;
    const double bin_size = get_double_from_int_or_default(sci_reduced_header, bin_size_tags, NAN) / angstrom_to_nm;

    return lamrms / cd * pix_size * bin_size;
}

static cpl_error_code
get_other_properties(const cpl_propertylist * plist_science_reduced,
        const cpl_propertylist * plist_ids, cpl_propertylist * global_header){

    const char * filter_name = "FREE";
    const char * filter_source_tag = "ESO INS FILT1 NAME";

    if(has_string_property(plist_science_reduced, filter_source_tag)){
        filter_name = cpl_propertylist_get_string(plist_science_reduced, filter_source_tag);
    }
    cpl_propertylist_update_string(global_header, "FILTER", filter_name);

    const double crder = calc_crder( plist_science_reduced, plist_ids);
    if(std::isnan(crder)) return CPL_ERROR_ILLEGAL_OUTPUT;

    const char * csyer_tag = "ESO DRS CSYER";
    const char * crder_tag = "ESO DRS CRDER";

    /* Positional errors (CSYER1 and CRDER1) cannot be written using the corresponding image keywords
     * because of FITS standard. Cannot write the corresponding table keywords because we don't have
     * a positional column. Hence use DRS keywords.*/
    cpl_propertylist_update_double(global_header, csyer_tag, 3.0);
    cpl_propertylist_set_comment(global_header, csyer_tag, "Systematic error in position [arcsec]");
    cpl_propertylist_update_double(global_header, crder_tag, crder);
    cpl_propertylist_set_comment(global_header, crder_tag, "Random error in position [arcsec]");
    if (cpl_error_get_code()) return cpl_error_get_code();

    return add_radec_err(plist_science_reduced, global_header);
}

static double get_ron_from_master_bias(fors_spec_idp_save_info& info){

    const cpl_frame * bias_frame =
                cpl_frameset_find_const(info.frameset, info.master_bias_tag);

    if(bias_frame == NULL) return NAN;

    cpl_propertylist * plist = cpl_propertylist_load(cpl_frame_get_filename(bias_frame), 0);

    if(plist == NULL) return NAN;

    double to_ret = NAN;

    const char * ron_tag = "ESO QC RON";

    if(has_double_property(plist, ron_tag)){
        to_ret = cpl_propertylist_get_double(plist, ron_tag);
    }

    cpl_propertylist_delete(plist);
    return to_ret;
}

void fors_spec_idp_save_idp_format(fors_spec_idp_save_info& info, const cpl_image * calibrated,
        const cpl_image * calibrated_error, const cpl_image * bkg,
        const cpl_image * reduced, const cpl_image * reduced_err,
        const cpl_array * wlens, const cpl_table * obj_table){

    if(calibrated == NULL || calibrated_error == NULL ||
            bkg == NULL || reduced == NULL || reduced_err == NULL
            || wlens == NULL || obj_table == NULL){
        cpl_msg_warning(cpl_func, "Unable to obtain some required images, aborting IDP generation");
        return;
    }

    char * pipe_id = get_pipe_id(info.iherit_frame);

    if(pipe_id == NULL){
        cpl_msg_warning(cpl_func, "Unable to extract instrument version, aborting IDP generation");
        return;
    }

    const double ron = get_ron_from_master_bias(info);

    if(std::isnan(ron)){
        cpl_msg_warning(cpl_func, "Unable to extract RON, aborting IDP generation");
        return;
    }

    if(std::isnan(info.spec_sye)){
        cpl_msg_warning(cpl_func, "Unable to extract SPEC_SYE, but continuing IDP generation");
    }

    std::vector<ra_dec> obj_ra_dec = get_ra_dec(obj_table);
    const cpl_size n_spectra = cpl_image_get_size_y(calibrated);

    const cpl_frame * sci_reduced_frame = cpl_frameset_find_const(info.frameset, info.reduced_science_tag);
    const cpl_frame * ids_frame = cpl_frameset_find_const(info.frameset, info.disp_coeff_tag);

    if(sci_reduced_frame == NULL){
            cpl_msg_warning(cpl_func, "Unable to extract frame from reduced science, aborting IDP generation");
            return;
    }

    if(ids_frame == NULL){
        cpl_msg_warning(cpl_func, "Unable to extract DISP_COEFF table, aborting IDP generation");
        return;
    }

    cpl_propertylist * plist = cpl_propertylist_load(cpl_frame_get_filename(sci_reduced_frame), 0);
    cpl_propertylist * plist_ids = cpl_propertylist_load(cpl_frame_get_filename(ids_frame), 0);
    cpl_propertylist * other_properties_plist = cpl_propertylist_new();
    cpl_propertylist * table_plist = cpl_propertylist_new();

    const cpl_error_code fail_props = get_other_properties(plist, plist_ids, other_properties_plist);

    if(plist == NULL || plist_ids == NULL || fail_props){
        cpl_msg_warning(cpl_func, "Unable to extract property list from reduced science or DISP_COEFF, aborting IDP generation");
        cpl_propertylist_delete(plist);
        cpl_propertylist_delete(plist_ids);
        cpl_propertylist_delete(other_properties_plist);
        cpl_propertylist_delete(table_plist);
        cpl_error_reset();
        return;
    }

    for(cpl_size idx_spectrum = 0; idx_spectrum < n_spectra; ++idx_spectrum){

        const double ra = obj_ra_dec[idx_spectrum].first;
        const double dec = obj_ra_dec[idx_spectrum].second;
        if(std::isnan(ra) || std::isnan(dec)){
            cpl_msg_warning(cpl_func, "Unable to extract ra/dec for spectrum %lld, aborting IDP generation", idx_spectrum + 1);
            break;
        }
        cpl_msg_info(cpl_func, "idx=%lld, RA=%f, Dec=%f", idx_spectrum, ra, dec);

        irplib_sdp_spectrum * spectrum = extract_1D_spectrum(calibrated, calibrated_error,
                bkg, reduced, reduced_err, wlens, idx_spectrum);


        if(spectrum == NULL || cpl_error_get_code()){
            cpl_msg_warning(cpl_func, "Unable to generate IDP spectrum %lld, aborting IDP generation", idx_spectrum + 1);
            cpl_error_reset();
            irplib_sdp_spectrum_delete(spectrum);
            break;
        }


        if(!add_spectrum_metadata(spectrum, wlens, plist, plist_ids,
                ra, dec, ron, info.spec_sye, info.exptime, info.det_exp_num,
                idx_spectrum)){
            cpl_msg_warning(cpl_func, "Unable to generate IDP metadata for spectrum %lld, aborting IDP generation", idx_spectrum + 1);
            cpl_error_reset();
            irplib_sdp_spectrum_delete(spectrum);
            break;
        }


        if(cpl_error_get_code()){
            irplib_sdp_spectrum_delete(spectrum);
            cpl_msg_warning(cpl_func, "Unable to load metadata for spectrum %lld, aborting IDP generation", idx_spectrum);
            cpl_error_reset();
            break;
        }

        const cpl_error_code fail = save_spectrum_as_idp(info, spectrum, other_properties_plist,
                table_plist, pipe_id, idx_spectrum + 1);

        irplib_sdp_spectrum_delete(spectrum);

        if(fail || cpl_error_get_code()){
            cpl_msg_warning(cpl_func, "Unable to save IDP spectrum %lld, aborting IDP generation", idx_spectrum);
            cpl_error_reset();
            break;
        }
    }

    cpl_propertylist_delete(plist);
    cpl_propertylist_delete(plist_ids);
    cpl_propertylist_delete(other_properties_plist);
    cpl_propertylist_delete(table_plist);
    cpl_free(pipe_id);
}

double fors_spec_idp_get_spec_sye(const cpl_table * offsets,
        const cpl_table * slits){

    if(offsets == NULL || slits == NULL) return NAN;

    const cpl_size nslits = cpl_table_get_nrow(slits);
    const cpl_size nlines = cpl_table_get_nrow(offsets);
    double off = 0;
    int num_offs = 0;
    for (cpl_size i = 0; i < nslits; i++) {
        int rej = 0;
        if (cpl_table_get_int(slits, "length", i, &rej) == 0 || rej == 1)
            continue;

        std::stringstream ss;
        ss<<"offset"<<cpl_table_get_int(slits, "slit_id", i, &rej);

        if(rej) continue;

        for(cpl_size j = 0; j < nlines; ++j){
            const double this_off = cpl_table_get_double(offsets, ss.str().c_str(), j, &rej);
            if(rej) continue;
            off += this_off;
            num_offs ++;
        }
    }

    if(cpl_error_get_code() || num_offs == 0)
    {
        cpl_error_reset();
        return NAN;
    }

    off = std::abs(off);
    return  off / (double)num_offs;
}

