/*                                                                            *
 *   This file is part of the ESO X-shooter Pipeline                          *
 *   Copyright (C) 2006 European Southern Observatory                         *
 *                                                                            *
 *   This library is free software; you can redistribute it and/or modify     *
 *   it under the terms of the GNU General Public License as published by     *
 *   the Free Software Foundation; either version 2 of the License, or        *
 *   (at your option) any later version.                                      *
 *                                                                            *
 *   This program is distributed in the hope that it will be useful,          *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of           *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the            *
 *   GNU General Public License for more details.                             *
 *                                                                            *
 *   You should have received a copy of the GNU General Public License        *
 *   along with this program; if not, write to the Free Software              *
 *   Foundation, 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA     *
 *                                                                            */

/*
 * $Author: amodigli $
 * $Date: 2011-12-02 14:13:14 $
 * $Revision: 1.3 $
 * $Name: not supported by cvs2svn $
 */
#ifdef HAVE_CONFIG_H
#  include <config.h>
#endif

/*--------------------------------------------------------------------------*/
/**
  @defgroup xsh_test_tools     
    Test some tools functions for performances check
  @ingroup unit_tests
*/
/*--------------------------------------------------------------------------*/
/**@{*/

/*---------------------------------------------------------------------------
                                Includes
 ---------------------------------------------------------------------------*/
#include <string.h>
#include <stdio.h>
#include <math.h>
#include <cpl.h>
#include <xsh_msg.h>
#include <xsh_utils.h>
#include <xsh_dfs.h>
#include <xsh_utils_table.h>

static cpl_error_code
xsh_line_group_stat(const int nbins, const int local_sz, const int nside,
                    const double* xbin, cpl_table* tab, double* local_median,
                    double* local_stdev, int* group_nbin, double* group_mean,
                    double* group_median)
{
    int j_lo=0;
    int j_hi=0;
    int ilo=0;
    int ihi=0;
    double wmin;
    double wmax;
    cpl_table* xtab;

    for (int i = 0; i < nbins; i++) {
        j_lo = (0 > (i - local_sz / 2)) ? 0 : (i - local_sz / 2);
        j_hi = ((j_lo + local_sz) < (nbins - 1)) ?
                        (j_lo + local_sz) : (nbins - 1);
        //xsh_msg_warning("j_lo=%d j_hi=%d",j_lo,j_hi);
        wmin = xbin[j_lo];
        wmax = xbin[j_hi];
        //xsh_msg_warning("wmin=%16.14g wmax=%16.14g nbins=%d",wmin,wmax,nbins);
        cpl_table_and_selected_double(tab, "WAVE", CPL_GREATER_THAN, wmin);
        cpl_table_and_selected_double(tab, "WAVE", CPL_LESS_THAN, wmax);
        xtab = cpl_table_extract_selected(tab);
        local_median[i] = cpl_table_get_column_median(xtab, "FLUX");
        local_stdev[i] = cpl_table_get_column_stdev(xtab, "FLUX");
        cpl_table_select_all(tab);
        xsh_free_table(&xtab);

        ilo = ((i - nside) > 0) ? (i - nside) : 0;
        ihi = ((i + nside) < (nbins - 1)) ? (i + nside) : nbins - 1;
        //xsh_msg_warning("ilo=%d ihi=%d",ilo,ihi);
        wmin = xbin[ilo];
        wmax = xbin[ihi];
        cpl_table_and_selected_double(tab, "WAVE", CPL_GREATER_THAN, wmin);
        cpl_table_and_selected_double(tab, "WAVE", CPL_LESS_THAN, wmax);
        xtab = cpl_table_extract_selected(tab);
        group_nbin[i] = ihi - ilo + 1;
        group_mean[i] = cpl_table_get_column_mean(xtab, "FLUX");
        group_median[i] = cpl_table_get_column_median(xtab, "FLUX");
        if(i<5) {
        xsh_msg_warning("nbin=%d mean=%g median=%g",
                        group_nbin[i],group_mean[i],group_median[i]);

        }
        cpl_table_select_all(tab);
        xsh_free_table(&xtab);
    }
    return cpl_error_get_code();
}

static cpl_error_code
xsh_line_fill_xbin_ybin_ysig(const int nbins, const double wmin,
                             const double bin, cpl_table* tab,
                             const int* h, double* xbin, double* ybin,
                             double* ysig)

{
    cpl_table* xtab;
    //ybin=cpl_array_get_data_double(ybin_array);
    double fct = 0;
    for (int i = 0; i < nbins; i++) {
        xbin[i] = wmin + bin * (i + 0.5);
        cpl_table_and_selected_double(tab, "WAVE", CPL_GREATER_THAN,
                        xbin[i] - bin);
        cpl_table_and_selected_double(tab, "WAVE", CPL_LESS_THAN,
                        xbin[i] + bin);
        xtab = cpl_table_extract_selected(tab);
        ybin[i] = cpl_table_get_column_median(xtab, "FLUX");

        fct = (h[i] > 0) ? h[i] : 1;
        //xsh_msg_warning("h=%d dh=%g fct=%g",h[i],(double)h[i],fct);
        ysig[i] = cpl_table_get_column_stdev(xtab, "FLUX");
        if(i<5) {
           xsh_msg_warning("ysig=%g",ysig[i]);
        }


        ysig[i] /= sqrt(fct);
        if(i<5) {
            xsh_msg_warning("Normalisation");
           xsh_msg_warning("ysig=%g",ysig[i]);
        }
        //xsh_msg_warning("xbin=%g",xbin[i]);
        //xsh_msg_warning("ybin=%g",ybin[i]);
        //xsh_msg_warning("ysig=%g",ysig[i]);
        cpl_table_select_all(tab);
        xsh_free_table(&xtab);
    }
    return cpl_error_get_code();
}

double*
xsh_find_em_lines(const int nbins, const int nside, const double min_bk_space,
                  const double intermed_bk_space, const double* local_median,
                  const double* local_stdev, const int* group_nbin,
                  const double* group_mean, const double* group_median,
                  int* pltype, double* bk_space_list)
{
    /* Groups that look like emission lines */
    double* em_lines_thresh = cpl_calloc(nbins, sizeof(double));
    for (int i = 0; i < nbins; i++) {
        em_lines_thresh[i] = local_median[i]
                        + 3. * local_stdev[i] / sqrt(group_nbin[i]);
        //xsh_msg_warning("em_lines_thresh[%d]=%g", i, em_lines_thresh[i]);
    }
    int n_em_lines = 0;
    for (int i = 0; i < nbins; i++) {
        if (((group_mean[i] >= em_lines_thresh[i])
                        && (group_median[i] >= local_median[i] + local_stdev[i]))) {
            pltype[i] = 1;
            n_em_lines++;
            xsh_msg_warning("em line i=%d",i);
        }
    }
    xsh_msg_warning("emission lines %d",n_em_lines);
    if (n_em_lines != 0) {
        for (int j = 0; j < nbins; j++) {
            /* adding extra pixels to the side of groups may cause ringing in
             * the sky fit at line edges */
            if (pltype[j] == 1) {
                int i = j;
                int ilo = ((i - nside) > 0) ? (i - nside) : 0;
                int ihi = ((i + nside) < (nbins - 1)) ? (i + nside) : nbins - 1;
                bk_space_list[i] = min_bk_space;
                for (int k = ilo; k < ihi; k++) {
                    bk_space_list[k] =
                                    (intermed_bk_space < bk_space_list[k]) ?
                                                    intermed_bk_space :
                                                    bk_space_list[k];
                }
            }
        }
    }
    return em_lines_thresh;
}

double*
xsh_find_abs_lines(const int nbins, const int nside, const double min_bk_space,
                   const double intermed_bk_space, const double* local_median,
                   const double* local_stdev, const int* group_nbin,
                   const double* group_mean, const double* group_median,
                   int* pltype, double* bk_space_list)
{
    /* Groups that look like absorbtion lines */
    double* abs_lines_thresh = cpl_calloc(nbins, sizeof(double));
    for (int i = 0; i < nbins; i++) {
        abs_lines_thresh[i] = local_median[i]
                        - 3. * local_stdev[i] / sqrt(group_nbin[i]);
        //xsh_msg_warning("em_lines_thresh[%d]=%g", i, em_lines_thresh[i]);
    }
    int n_abs_lines = 0;
    for (int i = 0; i < nbins; i++) {
        if (((group_mean[i] <= abs_lines_thresh[i])
                        && (group_median[i] <= local_median[i] + local_stdev[i]))) {
            pltype[i] = 2;
            n_abs_lines++;
        }
    }
    xsh_msg_warning("absorbtion lines %d",n_abs_lines);
    if (n_abs_lines != 0) {
        for (int j = 0; j < nbins; j++) {
            /* adding extra pixels to the side of groups may cause ringing in
             * the sky fit at line edges */
            if (pltype[j] == 2) {
                int i = j;
                int ilo = ((i - nside) > 0) ? (i - nside) : 0;
                int ihi = ((i + nside) < (nbins - 1)) ? (i + nside) : nbins - 1;
                bk_space_list[i] = min_bk_space;
                for (int k = ilo; k < ihi; k++) {
                    bk_space_list[k] =
                                    (intermed_bk_space < bk_space_list[k]) ?
                                                    intermed_bk_space :
                                                    bk_space_list[k];
                }
            }
        }
    }
    return abs_lines_thresh;
}

static cpl_error_code
xsh_find_spectrum_slope_changes(const int nbins, const int nside,
                                const double min_bk_space,
                                const double* ybin, const double* ysig,
                                int* pltype, double* bk_space_list)
{
    /*
     for(int i=0; i< nbins; i++) {
     xsh_msg_warning("bk_space=%g",bk_space_list[i]);
     }
     */
    /*
     xsh_msg_warning("bk_space=%g min_bk_space=%g intermed_bk_space=%g",
     bk_space,min_bk_space,intermed_bk_space);
     */
    /* other points where the slope appears to change rapidly */
    double* ker1 = cpl_calloc(3, sizeof(double));
    ker1[0] = -1;
    ker1[1] = 2;
    ker1[2] = -1;
    double* d2y = cpl_calloc(nbins, sizeof(double));
    memcpy(d2y, ybin, nbins * sizeof(double));
    for (int i = 1; i < nbins - 1; i++) {
        d2y[i] = ybin[i - 1] * ker1[0] + ybin[i] * ker1[1]
                        + ybin[i + 1] * ker1[1];
    }
    cpl_free(ker1);
    double* d2y_var = cpl_calloc(nbins, sizeof(double));
    double* yvar = cpl_calloc(nbins, sizeof(double));
    memcpy(yvar, ysig, nbins * sizeof(double));
    for (int i = 1; i < nbins - 1; i++) {
        yvar[i] *= yvar[i];
    }
    double* ker2 = cpl_calloc(3, sizeof(double));
    ker2[0] = 1;
    ker2[1] = 2;
    ker2[2] = 1;
    for (int i = 1; i < nbins - 1; i++) {
        d2y_var[i] = yvar[i - 1] * ker2[0] + yvar[i] * ker2[1]
                        + yvar[i + 1] * ker2[1];
    }
    cpl_free(ker2);
    cpl_free(yvar);
    double* d2y_sig = cpl_calloc(nbins, sizeof(double));
    for (int i = 1; i < nbins - 1; i++) {
        d2y_sig[i] = sqrt(d2y_var[i]);
    }

    cpl_free(d2y_var);
    int n_slope_change = 0;
    for (int i = 0; i < nbins; i++) {
        if (fabs(d2y[i]) >= (4 * d2y_sig[i]) && pltype[i] == 0) {
            pltype[i] = 3;
            n_slope_change++;
        }
    }
    xsh_msg_warning("slope changes %d/%d",n_slope_change,nbins);
    cpl_free(d2y_sig);
    cpl_free(d2y);
    if (n_slope_change != 0) {
        for (int j = 0; j < nbins; j++) {
            /* adding extra pixels to the side of groups may cause ringing in
             * the sky fit at line edges */
            if (pltype[j] == 3) {
                int i = j;
                int ilo = ((i - nside) > 0) ? (i - nside) : 0;
                int ihi = ((i + nside) < (nbins - 1)) ? (i + nside) : nbins - 1;
                //bk_space_list[i] = min_bk_space;
                for (int k = ilo; k < ihi; k++) {
                    bk_space_list[k] =
                                    (min_bk_space < bk_space_list[k]) ?
                                                    min_bk_space :
                                                    bk_space_list[k];

                }
                //xsh_msg_warning("slope changes %g",bk_space_list[j]);
            }
        }
    }
    xsh_msg_warning("min_bk_space %g",min_bk_space);
    return cpl_error_get_code();
}

static cpl_error_code
xsh_lines_set_breakpoints(cpl_table* stab)
{
    double* wave = cpl_table_get_data_double(stab, "WAVE");
    double* flux = cpl_table_get_data_double(stab, "FLUX");
    double wmin = cpl_table_get_column_min(stab, "WAVE");
    double wmax = cpl_table_get_column_max(stab, "WAVE");
    int nrow=cpl_table_get_nrow(stab);
    xsh_msg_warning("nrow=%d",nrow);

    /* Place points break points. Attempt to identify and place extra break
     * points around skylines. x=wavelength, y=sky counts.
     * This method misses closely spaced sky lines.
     */
    double dispersion = 0.049439349;
    double bin = dispersion;
    int nbins = (wmax - wmin) / bin+1;
    xsh_msg_warning("nbins=%d",nbins);
    xsh_msg_warning("bin=%16.14g",bin);
    //xsh_msg_warning("wmin=%16.14g wmax=%16.14g nbins=%d",wmin,wmax,nbins);
    cpl_table* histogram = xsh_histogram(stab, "WAVE", nbins, wmin, wmax);
    cpl_table_save(histogram, NULL, NULL, "histogram.fits", CPL_IO_DEFAULT);
    double* xbin = cpl_calloc(nbins, sizeof(double));
    double* ybin = cpl_calloc(nbins, sizeof(double));
    double* ysig = cpl_calloc(nbins, sizeof(double));
    int* h = cpl_table_get_data_int(histogram, "HY");
    xsh_line_fill_xbin_ybin_ysig(nbins, wmin, bin, stab, h, xbin, ybin, ysig);
    xsh_msg_warning("xbin=%16.14g,%16.14g,%16.14g,%16.14g,%16.14g",xbin[0],xbin[1],xbin[2],xbin[nbins-2],xbin[nbins-1]);
    xsh_msg_warning("ybin=%16.14g,%16.14g,%16.14g,%16.14g,%16.14g",ybin[0],ybin[1],ybin[2],ybin[nbins-2],ybin[nbins-1]);
    xsh_msg_warning("h=%d,%d,%d,%d,%d,%d,%d",h[0],h[1],h[2],h[3],h[4],h[nbins-2],h[nbins-1]);


    /* compute local quantities for group pixels
     */
    int local_sz = nbins / 10.;
    int nside = 2;
    /* ideal number of bins per group */
    int bin_per_group = 2 * nside + 1;
    int* group_nbin = cpl_calloc(nbins, sizeof(int));
    double* group_mean = cpl_calloc(nbins, sizeof(double));
    double* group_median = cpl_calloc(nbins, sizeof(double));
    double* local_median = cpl_calloc(nbins, sizeof(double));
    double* local_stdev = cpl_calloc(nbins, sizeof(double));
    double* bk_space_list = (double*) cpl_calloc(nbins, sizeof(double));
    xsh_line_group_stat(nbins, local_sz, nside, xbin, stab, local_median,
                        local_stdev, group_nbin, group_mean, group_median);


    cpl_table* lines_tab=cpl_table_new(nbins);
    cpl_table_wrap_double(lines_tab,group_mean,"YBIN");
    cpl_table_wrap_int(lines_tab,group_nbin,"GNBIN");
    cpl_table_wrap_double(lines_tab,group_mean,"GMEDIAN");
    cpl_table_wrap_double(lines_tab,group_mean,"GMEAN");
    cpl_table_wrap_double(lines_tab,local_median,"LMEDIAN");
    cpl_table_wrap_double(lines_tab,local_stdev,"LSTDEV");
    cpl_table_wrap_double(lines_tab,bk_space_list,"BKSPACE");
    cpl_table_new_column(lines_tab,"LTYPE",CPL_TYPE_INT);

    cpl_table_fill_column_window_int(lines_tab,"LTYPE",0,nbins,0);
    int* pltype=cpl_table_get_data_int(lines_tab,"LTYPE");
    double* pbk_space_list=cpl_table_get_data_double(lines_tab,"BKSPACE");
    int status=0;  /* make sure bk_space is > min_bk_space */
    double bk_space = 0.015262156;
    double min_bk_space = 0.0025436927;
    bk_space = (bk_space > min_bk_space) ? bk_space : min_bk_space;
    /* Buffer pixels at the edges of lines and in slowly varying regions
      * with break points at the intermediate spacing
      */
    double intermed_bk_space = min_bk_space + 0.3 * (bk_space - min_bk_space);
    xsh_msg_warning("bk_space=%g",bk_space);
    /* Default is the max break point spacing */

    for(int i=0;i<nbins; i++) {
        pbk_space_list[i] = bk_space;

    }

    /* Groups that look like emission lines */
    double* em_lines_thresh = xsh_find_em_lines(nbins, nside, min_bk_space,
                    intermed_bk_space, local_median, local_stdev, group_nbin,
                    group_mean, group_median, pltype, pbk_space_list);
    cpl_table_save(lines_tab,NULL,NULL,"lines_tab.fits",CPL_IO_DEFAULT);

    /* Groups that look like absorbtion lines */
    double* abs_lines_thresh = xsh_find_abs_lines(nbins, nside, min_bk_space,
                    intermed_bk_space, local_median, local_stdev, group_nbin,
                    group_mean, group_median, pltype, pbk_space_list);

    /*
    for(int i=0; i< nbins; i++) {
        xsh_msg_warning("bk_space=%g",bk_space_list[i]);
    }
    */

    /*
    xsh_msg_warning("bk_space=%g min_bk_space=%g intermed_bk_space=%g",
                    bk_space,min_bk_space,intermed_bk_space);
     */

    /* other points where the slope appears to change rapidly */
    xsh_find_spectrum_slope_changes(nbins, nside, min_bk_space, ybin, ysig,
                    pltype, pbk_space_list);

    /* additional regions that do not appear constant */
    xsh_print_rec_status(0);
    int region_nside = 10;
    cpl_table* xtab=NULL;
    for (int i = 0; i < nbins; i++) {
        xsh_print_rec_status(1);
         int ilo = ((i - region_nside) > 0) ? (i - region_nside) : 0;
         int ihi = ((i + region_nside) < (nbins - 1)) ? (i + region_nside) : (nbins - 1);
         //xsh_msg_warning("ilo=%d ihi=%d",ilo,ihi);
         wmin = xbin[ilo];
         wmax = xbin[ihi];
         xsh_print_rec_status(2);
         cpl_table_dump_structure(lines_tab,stdout);
         cpl_table_and_selected_double(lines_tab, "WAVE", CPL_GREATER_THAN, wmin);
         xsh_print_rec_status(3);
         cpl_table_and_selected_double(lines_tab, "WAVE", CPL_LESS_THAN, wmax);
         xsh_print_rec_status(4);
         xtab = cpl_table_extract_selected(lines_tab);
         xsh_print_rec_status(5);
         double* region_bkspace = NULL;
         int nrow = cpl_table_get_nrow(xtab) ;
         double* pregion_bkspace = cpl_table_get_data_double(lines_tab,"BKSPACE");
         xsh_print_rec_status(6);
         double* pregion_ybin = cpl_table_get_data_double(lines_tab,"YBIN");
         xsh_print_rec_status(7);
         for (int j = 0; j< nrow; j++ ) {
             // Bins were spacing is still the maximum
             int n_max=0;
             if ( pregion_bkspace[j] == bk_space )  {
                 pltype[j] = 4;
                 n_max++;
             }
             cpl_table_and_selected_int(xtab,"LTYPE",CPL_EQUAL_TO,4);
             cpl_table* stab=cpl_table_extract_selected(xtab);
             if(n_max >= region_nside) {
                 // determine weather scatter goes down with the binning
                 double stdev_bin = cpl_table_get_column_stdev(stab,"YBIN");
                 double stdev_group = cpl_table_get_column_stdev(stab,"GMEAN");
                 if( stdev_bin/stdev_group <= 0.7 * sqrt (bin_per_group) ) {
                     // scatter not decreasing with binning as expected for a
                     // constant sky bkg

                     for (int k = ilo; k< ihi; k++ ) {
                         pltype[k] = 5;
                         pbk_space_list[k] =  ( (pregion_bkspace[k] < intermed_bk_space) )? pregion_bkspace[k] : intermed_bk_space;
                     }
                 }
             }
             xsh_free_table(&stab);
             cpl_table_select_all(lines_tab);
             xsh_free_table(&xtab);
             xsh_print_rec_status(3);
             //exit(0);
         }

     }


    exit(0);




/*

    for(int i=0; i< nbins; i++) {
        xsh_msg_warning("bk_space=%g",bk_space_list[i]);
    }
*/


    /* free memory */
    cpl_table_unwrap(lines_tab,"YBIN");
    cpl_table_unwrap(lines_tab,"GNBIN");
    cpl_table_unwrap(lines_tab,"GMEDIAN");
    cpl_table_unwrap(lines_tab,"GMEAN");
    cpl_table_unwrap(lines_tab,"LMEDIAN");
    cpl_table_unwrap(lines_tab,"LSTDEV");
    cpl_table_unwrap(lines_tab,"BKSPACE");
    xsh_free_table(&lines_tab);


    cpl_free(xbin);
    cpl_free(ybin);
    cpl_free(ysig);
    cpl_free(group_nbin);
    cpl_free(group_mean);
    cpl_free(group_median);
    cpl_free(local_median);
    cpl_free(local_stdev);
    cpl_free(bk_space_list);
    cpl_free(em_lines_thresh);
    cpl_free(abs_lines_thresh);


    xsh_msg_warning("ok1");
    xsh_free_table(&histogram);
    return cpl_error_get_code();
}

/*---------------------------------------------------------------------------
                            Defines
 ---------------------------------------------------------------------------*/
/*--------------------------------------------------------------------------*/
/**
  @brief
    Some test about performances
  @return
 */
/*--------------------------------------------------------------------------*/
int main(int argc, char **argv)
{

    char *fname = NULL;
    char *arm = NULL;
    cpl_test_init(PACKAGE_BUGREPORT,CPL_MSG_WARNING);

    if (argc > 1){
        fname = argv[1];
    }
    else{
        return 0;
    }
    xsh_msg_warning("fname=%s",fname);
    cpl_table* tab=cpl_table_load(fname,1,0);
    xsh_lines_set_breakpoints(tab);
    xsh_free_table(&tab);
    return cpl_test_end(0);
}


/**@}*/
