/*                                                                              *
 *   This file is part of the ESO UVES Pipeline                                 *
 *   Copyright (C) 2004,2005 European Southern Observatory                      *
 *                                                                              *
 *   This library is free software; you can redistribute it and/or modify       *
 *   it under the terms of the GNU General Public License as published by       *
 *   the Free Software Foundation; either version 2 of the License, or          *
 *   (at your option) any later version.                                        *
 *                                                                              *
 *   This program is distributed in the hope that it will be useful,            *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of             *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the              *
 *   GNU General Public License for more details.                               *
 *                                                                              *
 *   You should have received a copy of the GNU General Public License          *
 *   along with this program; if not, write to the Free Software                *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA       *
 *                                                                              */
 
/*
 * $Author: amodigli $
 * $Date: 2013-08-08 13:36:46 $
 * $Revision: 1.39 $
 * $Name: not supported by cvs2svn $
 *
 */

/*----------------------------------------------------------------------------*/
/**
 * @addtogroup uves_response
 */
/*----------------------------------------------------------------------------*/

#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include <uves_response_efficiency.h>
#include <uves_response_utils.h>
#include <uves_reduce.h>
#include <uves_reduce_utils.h>
#include <uves_dfs.h>
#include <uves_pfits.h>
#include <uves_wavecal_utils.h>
#include <uves_utils_polynomial.h>
#include <uves_utils.h>
#include <uves_utils_wrappers.h>
#include <uves_utils_cpl.h>
#include <uves.h>
#include <uves_error.h>
#include <uves_msg.h>

#include <cpl.h>

#include <stdbool.h>

/*-----------------------------------------------------------------------------
                                Defines
 -----------------------------------------------------------------------------*/
#define H_BAR 6.626068e-34           /* SI-units */
#define PRIMARY_DIA 818              /* Primary diameter (cm) */
#define OBSTR_DIA   140              /* Central obstruction diameter (cm) */
#define TELESCOPE_EFFECTIVE_AREA   \
  (M_PI * (PRIMARY_DIA * PRIMARY_DIA - OBSTR_DIA * OBSTR_DIA) / 4.0)  /* (cm^2) */

/**@{*/
/**
  @brief    Calculate quantum detection efficiency
  @param    raw_image           The raw image
  @param    raw_header          FITS header of raw image
  @param    rotated_header      Header describing the geometry of the raw image after
                                rotation and removal of pre- and overscan areas
  @param    master_bias         The master bias image for this chip, or NULL
  @param    master_dark         The master dark image for this chip, or NULL
  @param    mdark_header        FITS header of master dark frame
  @param    ordertable          Order table describing the order locations on the raw image
  @param    order_locations     The polynomial describing the order positions
  @param    linetable           Length 3 array of linetable for sky, object, sky.
  @param    linetable_header    Length 3 array of linetable headers for sky, object, sky.
  @param    dispersion_relation Length 3 array of dispersion relations for sky, object, sky.
  @param    flux_table          Catalogue standard star fluxes
  @param    atm_extinction      Atmospheric extinction coefficients
  @param    chip                CCD chip
  @param    debug_mode               If set to true, intermediate results are saved to 
                                the current directory
  @param    parameters          The recipe parameter list containing parameters
                                for background subtraction, flat-fielding, extraction, rebinning
  @param    PACCURACY           Pointing accuracy (in arcseconds) used to identify object
  @param    efficiency          (out) The quantum detection efficiency table
  @param    blaze_efficiency    (out) Efficiency at blaze function maximum, for each order
  @return   The reduced spectrum
  
**/
/*----------------------------------------------------------------------------*/
cpl_error_code
uves_response_efficiency(const cpl_image *raw_image, 
                         const uves_propertylist *raw_header, 
                         const uves_propertylist *rotated_header,
                         const cpl_image *master_bias,
                         const cpl_image *master_dark, 
                         const uves_propertylist *mdark_header, 
                         const cpl_table *ordertable, 
                         const polynomial *order_locations,
                         const cpl_table *linetable[3], 
                         const uves_propertylist *linetable_header[3], 
                         const polynomial *dispersion_relation[3],
                         const cpl_table *flux_table,
                         const cpl_table *atm_extinction,
                         enum uves_chip chip,
                         bool debug_mode,
                         const cpl_parameterlist *parameters,
                         /* Identification */
                         double PACCURACY,
                         /* Output */
                         cpl_table **efficiency,
                         cpl_table **blaze_efficiency)
{
    cpl_image *background             = NULL;
    cpl_image *rebinned_spectrum      = NULL;
    cpl_image *rebinned_noise         = NULL;
    cpl_image *merged_sky             = NULL;
    cpl_image *merged_spectrum        = NULL;
    cpl_image *merged_noise           = NULL;
    cpl_image *reduced_spectrum       = NULL;
    cpl_image *reduced_noise          = NULL;
    cpl_image *reduced_rebinned       = NULL;
    cpl_image *reduced_rebinned_noise = NULL;
    uves_propertylist *rebinned_header = NULL;
    uves_propertylist *reduced_header  = NULL;
    polynomial *disprel_1d            = NULL;

    cpl_image *response_orders     = NULL;
    cpl_image *efficiency_spectrum = NULL;
    cpl_table *central_efficiency  = NULL;

    cpl_table *info_tbl            = NULL;  /* Local. The info tbl
                           should not be calculated twice
                           in the response recipe. It is
                           calculated in the first extraction */

    char *ref_obj_id = NULL;
    
    double wavestep;
    double extraction_slit;
    cpl_image* wave_map=NULL;
    cpl_image* reduced_rebinned_no_bpm=NULL;
    cpl_mask* bpm=NULL;

    /* Set parameters
       wavestep       = 10 * 2/3 * mean(pixelsize)

       After execution, revert to current value
    */
    {
    double smooth_step;
    
    /* wavestep */
    check( uves_get_parameter(parameters, NULL, 
                  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep", 
                  CPL_TYPE_DOUBLE, &wavestep),
           "Error getting resampling step size");
    
    check( smooth_step = cpl_table_get_column_mean(linetable[1], LINETAB_PIXELSIZE),
           "Error reading mean pixelsize");
    
    smooth_step = 10*2*smooth_step/3;
        
    /* Cast to non-const is okay. The parameter is reset to its previous value
       (see below), so there is not net change (unless the reduction fails, 
       in which case parameter list will change).
    */
    check( uves_set_parameter((cpl_parameterlist *) parameters,
                  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep",
                  CPL_TYPE_DOUBLE, &smooth_step),
           "Error setting resampling step size");
    }
    
    check( uves_reduce(raw_image, 
                       raw_header, 
                       rotated_header,
               master_bias,
               master_dark, 
                       mdark_header,
               NULL, 
                       NULL,                              /* No master flat */
               ordertable, 
                       order_locations,
               linetable, 
                       linetable_header, 
                       dispersion_relation,
               chip,
               debug_mode, 
               parameters, 
                       make_str(UVES_RESPONSE_ID),
                       ".efficiency",
               /* Output */
               NULL, 
                       NULL, 
                       NULL,                        /* 2d products */
               NULL,                        /* Cosmic ray table */
               &wave_map,
               &background,
               NULL, 
                       NULL,                        /* Variance of flat-fielded spectrum */
               NULL, 
                       NULL,                        /* Don't need these 
                               intermediate products */
               &merged_sky,
               &rebinned_spectrum, 
                       &rebinned_noise, 
                       &rebinned_header,
               &merged_spectrum, 
                       &merged_noise, 
                       &reduced_header,
               &reduced_rebinned, 
                       &reduced_rebinned_noise,
               &reduced_spectrum, 
                       &reduced_noise,
                       &info_tbl,
               &extraction_slit,   /* not passed on to the caller */
                       NULL),
       "Could not reduce frame");
    
    /* Reset parameter to previous value */
    {
    uves_msg_debug("Resetting parameter wavestep = %e", wavestep);

    /* Cast to non-const is okay. There is no net change in the parameter list (see above). */
    check( uves_set_parameter((cpl_parameterlist *) parameters, 
                  make_str(UVES_RESPONSE_ID) "", "reduce.rebin.wavestep", 
                  CPL_TYPE_DOUBLE, &wavestep),
           "Error resetting resampling step size");
    }
    
    
    /* Save reduced spectrum */
    if (debug_mode) 
    {
        /* Window number doesn't apply. This is middle window minus two other (sky) windows */
        check( uves_save_image_local("Reduced spectrum (2d)", "reduced", 
                     reduced_rebinned, chip, -1, -1, rebinned_header, true), 
           "Error saving reduced spectrum (2d)");
        
        check( uves_save_image_local("Reduced spectrum (2d) noise", "errreduced", 
                     reduced_rebinned_noise, chip, -1, -1, rebinned_header, true),
           "Error saving reduced spectrum (2d) noise");

        check( uves_save_image_local("Reduced spectrum", "merged", 
                     reduced_spectrum, chip, -1, -1, reduced_header, true), 
           "Error saving reduced spectrum");
        
        check( uves_save_image_local("Reduced spectrum noise", "errmerged", 
                     reduced_noise, chip, -1, -1, reduced_header, true), 
           "Error saving reduced spectrum noise");
    }
    
    uves_msg("Dividing by catalogue flux");
    /*
     * Calculate 2d response curve  (don't scale to unit exposure time, binning, gain, ... ) 
     */
    /* in some cases are flagged too many bad pixels on sky frames used 
       to compute reduced object in linear extraction method, this 
       affecting the number of bad pixels of the reduced_rebinned obj spectrum
       to fix this we erase the associated bad pixel map
    */
    reduced_rebinned_no_bpm=cpl_image_duplicate(reduced_rebinned);
    bpm=cpl_image_unset_bpm(reduced_rebinned_no_bpm);
    check( response_orders = uves_calculate_response(reduced_rebinned_no_bpm, 
                                                     rebinned_header,
                                                     flux_table,
                                                     raw_header, PACCURACY,
                                                     false,
                                                     &ref_obj_id),/*  flux/std_flux  */
       "Could not calculate response curve");

    uves_free_image(&reduced_rebinned_no_bpm);
    uves_free_mask(&bpm);

    if (debug_mode)
    {
        check( uves_save_image_local("2d response curve", "resp",
                     response_orders, chip, -1, -1, rebinned_header, true),
           "Error saving 2d response curve");
    }

    /*
     *   Extinction correction, exposure time + gain
     */
    {
    int n_traces = cpl_image_get_size_y(merged_spectrum);    /* Number of spatial traces */
    
    assure( n_traces == 1, CPL_ERROR_ILLEGAL_INPUT,
        "2d extraction/reduction not supported");
    
    check( efficiency_spectrum = uves_normalize_spectrum(response_orders, NULL,
                                 /* Spectrum, noise */
                                 rebinned_header,
                                 raw_header,
                                 n_traces,
                                 chip,
                                 atm_extinction,
                                 false,  /* Don't divide by binning */
                                 NULL),  /* Don't need output noise */
           "Could not normalize spectrum");
    }

    /*
     *   7 x 1 median filter
     */
    uves_msg("Applying 7x1 median filter");
    check( uves_filter_image_median(&efficiency_spectrum, 3, 0, false),
       "Error applying median filter");

    
    uves_msg("Calculating quantum detection efficiency");

    {
    int nx, nbins, norders, order;
    int first_abs_order, last_abs_order, abs_order;  /* Absolute order numbers */
    double dlambda;
    double average_noise;               /* Median of noise of rebinned spectrum */
    int row = 0;                                       /* Next unused table row */

    double *efficiency_data;               /* For efficiency. cpl_image_get()   */
    double *reduced_noise_data;            /* is slow when there are bad pixels */

    efficiency_data    = cpl_image_get_data_double(efficiency_spectrum);
    reduced_noise_data = cpl_image_get_data_double(reduced_rebinned_noise);

    nx      = cpl_image_get_size_x(raw_image);
    nbins   = cpl_image_get_size_x(efficiency_spectrum);
    norders = cpl_image_get_size_y(efficiency_spectrum);
    
    *efficiency = cpl_table_new(nbins * norders);
    cpl_table_new_column(*efficiency, "Wave", CPL_TYPE_DOUBLE);
    cpl_table_new_column(*efficiency, "Eff", CPL_TYPE_DOUBLE);
    cpl_table_new_column(*efficiency, "Binsize", CPL_TYPE_DOUBLE);
    cpl_table_new_column(*efficiency, "Order", CPL_TYPE_INT);
    row = 0;
    
    check( first_abs_order = uves_pfits_get_firstabsorder(linetable_header[1]), 
           "Could not read order numbers from line table header");
    check( last_abs_order  = uves_pfits_get_lastabsorder (linetable_header[1]), 
           "Could not read order numbers from line table header");
    
    check( dlambda = uves_pfits_get_cdelt1(rebinned_header),
           "Error reading bin width from header");
    
    check( average_noise = cpl_image_get_median(reduced_rebinned_noise),
           "Error reading median noise level");

    for (order = 1; order <= norders; order++)
        {
        double lambda_start, lambda, lambda_end;
        double x;
        int bin;

        abs_order = uves_absolute_order(first_abs_order, last_abs_order, order);

        check( lambda_start = uves_pfits_get_wstart(rebinned_header, order),
               "Error reading start wavelength from header");
        
        check( lambda_end   = uves_pfits_get_wend(rebinned_header, order),
               "Error reading end wavelength from header");

        /* Get 1d dispersion relation for this order
           f_1d = f(x, m=abs_order) 

           Collapsing a polynomial is slow, so do it
           only once per order
        */
        uves_polynomial_delete(&disprel_1d);
        check( disprel_1d = uves_polynomial_collapse(dispersion_relation[1],
                                 2,  /* Independent variable number */
                                 abs_order),
               "Error getting 1d dispersion relation for absolute order #%d", abs_order);

        x = 1;
        for (lambda = lambda_start, bin = 1;
             lambda < lambda_end + 0.5 * dlambda && bin <= nbins; 
             bin++, lambda += dlambda)
            {
            double flux;
            double dldx;
            double noise;            /* Only use positions with low noise 
                            (middle of blaze function) */
            
            /* flux  = cpl_image_get(efficiency_spectrum, 
                                     bin, order, &pis_rejected);
               noise = cpl_image_get(reduced_rebinned_noise, 
               bin, order, &pis_rejected); */
            flux  = efficiency_data   [(bin-1) + (order-1) * nbins];
            noise = reduced_noise_data[(bin-1) + (order-1) * nbins];


            /*
             *   Energy per (time * area * wavelength) =
             *    ((electron counts)/gain) * (hc/lambda) / 
             *    (time * area * |dlambda/dx|)
             *
             *   We already divided by exposure time, gain
             *   We did not multiply by dlambda/dx during rebinning,
             *    so now is the time to do it
             */

            /* Solve  f(x,m) = m*lambda  for x.
             *
             * This is equivalent to solving f_1d(x) = m*lambda
            */
            
            check( x = uves_polynomial_solve_1d(
                   disprel_1d, 
                   abs_order * lambda, /* right hand side    */
                   x,                  /* guess              */
                   1),                 /* multiplicity       */
                   "Could not solve dispersion relation for x "
                   "at (m, lambda) = (%d, %f)", abs_order, lambda);
            


            /*  For constant absolute order number, m:
                dl/dx  =  d (l.m)/dx / m  */
            
            check( dldx = fabs(uves_polynomial_derivative_2d(
                           dispersion_relation[1], 
                           x, 
                           abs_order, 1) / abs_order),
                   "Could not evaluate dispersion relation");
            
                        /* Don't make a linear interpolation
            weight = (lambda - lambda_start) / (lambda_end - lambda_start);
            
            check( dldx =
                   fabs(uves_polynomial_derivative_2d(
                    dispersion_relation[1],
                    1 , abs_order,
                    1) / abs_order) * (1 - weight) +
                   fabs(uves_polynomial_derivative_2d(
                    dispersion_relation[1],
                    nx, abs_order, 1) / abs_order) * weight,
                   "Could not evaluate dispersion relation");
            */

            flux = flux * 1e16 * 1e17 * H_BAR * SPEED_OF_LIGHT /
                (dldx * lambda * TELESCOPE_EFFECTIVE_AREA);
            /* The factor 1e17 accounts for the conversion Joule<-erg
             * (10^7) and Angstrom->meters (10^10)
             * The factor 1e16 is to correct for the fact that the
             * catalogue flux is in units of (10^-16 <standard units>)
             */

            if (noise < 3*average_noise)
                {
                check(( cpl_table_set_double(*efficiency, "Wave", row, lambda),
                    cpl_table_set_double(*efficiency, "Eff",  row, flux),
                    cpl_table_set_double(*efficiency, "Binsize",  row, dldx),
                    cpl_table_set_int   (*efficiency, "Order",  row, order),
                    row++),
                       "Error updating efficiency table row %d", row);
                }
            }
        }
    
    /* Remove unused rows of efficiency table */
    check( cpl_table_set_size(*efficiency, row), 
           "Error setting size of efficiency table to %d rows", row);
    /* remove negative efficiency points */
    cpl_table* tmp=cpl_table_duplicate(*efficiency);
    row=cpl_table_and_selected_double(tmp,"Eff",CPL_GREATER_THAN,0.);
    uves_free_table(efficiency);
    *efficiency=cpl_table_extract_selected(tmp);
    uves_free_table(&tmp);


    /* Get the "top efficiency" (90% percentile efficiency of middle 20% of each order) */

    *blaze_efficiency = cpl_table_new(norders);
    cpl_table_new_column(*blaze_efficiency, "Order", CPL_TYPE_INT);
    cpl_table_new_column(*blaze_efficiency, "Wave" , CPL_TYPE_DOUBLE);
    cpl_table_new_column(*blaze_efficiency, "Eff"  , CPL_TYPE_DOUBLE);
    row = 0;
    
    for (order = 1; order <= norders; order++)
        {
        double lambda_min;
        double lambda_central_min;
        double lambda_central;
        double lambda_central_max;
        double lambda_max;
        double top_efficiency;

        abs_order = uves_absolute_order(first_abs_order, last_abs_order, order);


        check( lambda_min = uves_pfits_get_wstart(rebinned_header, order),
               "Error reading bin width from header");

        check( lambda_max = uves_pfits_get_wend(rebinned_header, order),
               "Error reading bin width from header");
        
        lambda_central_min = lambda_min + 0.4 * (lambda_max - lambda_min);
        lambda_central     = lambda_min + 0.5 * (lambda_max - lambda_min);
        lambda_central_max = lambda_min + 0.6 * (lambda_max - lambda_min);
        
        /* Select rows in this order in range 
           ]lambda_central_min ; lambda_central_max[ */
        cpl_table_select_all(*efficiency);
        cpl_table_and_selected_int   (*efficiency, "Order", 
                          CPL_EQUAL_TO    , order);
        cpl_table_and_selected_double(*efficiency, "Wave" , 
                          CPL_GREATER_THAN, lambda_central_min);
        cpl_table_and_selected_double(*efficiency, "Wave" , 
                          CPL_LESS_THAN   , lambda_central_max);
        
        uves_msg_debug("%" CPL_SIZE_FORMAT " bins in central 20 %% range of order #%d",
                   cpl_table_count_selected(*efficiency), order);
        
        if ( cpl_table_count_selected(*efficiency) > 0)
            {
            uves_free_table(&central_efficiency);
            central_efficiency = cpl_table_extract_selected(*efficiency);
            
            /* Get 90% percentile efficiency */
            uves_sort_table_1(central_efficiency, "Eff", false);     /* Ascending */
            
            top_efficiency = cpl_table_get_double(
                central_efficiency, "Eff", 
                (int) (0.9 * cpl_table_get_nrow(central_efficiency)), NULL);
            }
        else
            {
            uves_msg_debug("No wavelength bins in central 20%% range of order #%d",
                       order);
            top_efficiency = 0;
            }
        
        uves_msg("Efficiency(lambda = %.2f A) = %.2f%%", 
             lambda_central, top_efficiency*100);

        check(( cpl_table_set_int   (*blaze_efficiency, "Order", row, order),
            cpl_table_set_double(*blaze_efficiency, "Wave" , row, lambda_central),
            cpl_table_set_double(*blaze_efficiency, "Eff"  , row, top_efficiency),
            row++),
               "Error updating blaze efficiency table");
        } /* for order */
    }

  cleanup:
    uves_free_image(&background);
    uves_free_image(&rebinned_spectrum);
    uves_free_image(&rebinned_noise);
    uves_free_image(&merged_sky);
    uves_free_image(&merged_spectrum);
    uves_free_image(&merged_noise);
    uves_free_image(&reduced_spectrum);
    uves_free_image(&reduced_noise);
    uves_free_image(&reduced_rebinned);
    uves_free_image(&reduced_rebinned_noise);
    uves_free_propertylist(&reduced_header);
    uves_free_propertylist(&rebinned_header);
    uves_polynomial_delete(&disprel_1d);

    uves_free_image(&response_orders);
    uves_free_image(&efficiency_spectrum);
    uves_free_table(&central_efficiency);
    uves_free_table(&info_tbl);
    
    cpl_free(ref_obj_id);

    if (cpl_error_get_code() != CPL_ERROR_NONE)
    {
        uves_free_table(efficiency);
        uves_free_table(blaze_efficiency);
    }
    
    return cpl_error_get_code();
}
/**@}*/
