/* $Id: visir_img_ff.c,v 1.94 2010-03-09 12:52:18 llundin Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: llundin $
 * $Date: 2010-03-09 12:52:18 $
 * $Revision: 1.94 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include <string.h>

#include "visir_recipe.h"
#include "irplib_flat.h"

/*-----------------------------------------------------------------------------
                            Defines
 -----------------------------------------------------------------------------*/

#define PAFCOPY "^(DATE-OBS|ARCFILE|ESO TPL ID|ESO DET DIT|MJD-OBS)$"
#define PAF_IMG "^(ESO INS FILT1 NAME|ESO INS PFOV)$"
#define PAF_SPC "^(ESO INS FILT2 NAME|ESO INS RESOL|ESO INS SLIT1 WID" \
                 "|ESO INS GRAT1 WLEN|ESO INS GRAT1 NAME)$"

#ifndef VISIR_IMG_FF_BORDER
#define VISIR_IMG_FF_BORDER 64
#endif

/* Different powers of two */
#define visir_ff_mode_default (1<<0)
#define visir_ff_mode_spc    (1<<1)
#define visir_ff_mode_tech    (1<<2)

#define RECIPE_STRING "visir_img_ff"

/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

static cpl_imagelist * visir_img_ff_reduce(const irplib_framelist *);

static cpl_error_code visir_img_ff_qc(cpl_propertylist *,
                                      cpl_propertylist *,
                                      const irplib_framelist *);

static cpl_error_code visir_img_ff_save(cpl_frameset *,
                                        const cpl_parameterlist *,
                                        const cpl_propertylist *,
                                        const cpl_propertylist *,
                                        const cpl_imagelist *,
                                        const cpl_image *);

VISIR_RECIPE_DEFINE(visir_img_ff, VISIR_PARAM_LOWLIM | VISIR_PARAM_HIGHLIM,
                    "Flat field recipe",
                    "This recipe computes the flatfield.\n"
                    "The files listed in the Set Of Frames (sof-file) must be "
                    "tagged either\n"
                    "VISIR-flatfield-raw-file.fits " VISIR_IMG_FF_RAW " or\n"
                    "VISIR-flatfield-raw-file.fits " VISIR_IMG_TECH_FF_RAW
                    " or\n"
                    "VISIR-flatfield-raw-file.fits " VISIR_SPC_FF_RAW " or\n"
                    "VISIR-flatfield-raw-file.fits " VISIR_SPC_TECH_FF_RAW "\n"
                    "\n"
                    "The corresponding primary product will have a FITS card\n"
                    "'HIERARCH ESO PRO CATG' with a value of\n"
                    VISIR_IMG_FF_PROCATG " or\n"
                    VISIR_IMG_TECH_FF_PROCATG " or\n"
                    VISIR_SPC_FF_PROCATG " or\n"
                    VISIR_SPC_TECH_FF_PROCATG
                    "\n");

/*-----------------------------------------------------------------------------
                            Static variables
 -----------------------------------------------------------------------------*/

const unsigned visir_img_ff_border = VISIR_IMG_FF_BORDER;

enum _visir_ff_mode_ {
    visir_ff_none     = 0,
    visir_ff_img      = visir_ff_mode_default,
    visir_ff_spc      = visir_ff_mode_default | visir_ff_mode_spc,
    visir_ff_img_tech = visir_ff_mode_default | visir_ff_mode_tech,
    visir_ff_spc_tech = visir_ff_mode_default | visir_ff_mode_tech
                      | visir_ff_mode_spc
};

typedef enum _visir_ff_mode_ visir_ff_mode;

static struct {
    /* Inputs */
    visir_ff_mode img_mode;
    double        bpm_lo_thresh;
    double        bpm_hi_thresh;
    /* Outputs */
    const char  * procatg_ff;
    const char  * procatg_bpm;
    int           bpm_nb_bad;
    cpl_boolean   is_lamp;
    double        fp_noise;  /* The Fixed Pattern Noise */
    double        lamp_flux; /* The Lamp Flux (intended for a pair of
                                lamp-on/lamp-off images) */
} visir_img_ff_config;

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_img_ff   Flat field
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/
/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    parlist     the parameters list
  @param    framelist   the frames list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_img_ff(cpl_frameset            * framelist,
                        const cpl_parameterlist * parlist)
{
    irplib_framelist * allframes = NULL;
    irplib_framelist * rawframes = NULL;
    cpl_propertylist * qclist    = cpl_propertylist_new();
    cpl_propertylist * paflist   = cpl_propertylist_new();
    cpl_imagelist    * flat      = NULL;
    cpl_image        * bpm_int   = NULL;
    cpl_mask         * bpm       = NULL;
    

    /* Retrieve input parameters */
    visir_img_ff_config.bpm_lo_thresh = 
        visir_parameterlist_get_double(parlist, RECIPE_STRING, VISIR_PARAM_LOWLIM);
    visir_img_ff_config.bpm_hi_thresh = 
        visir_parameterlist_get_double(parlist, RECIPE_STRING, VISIR_PARAM_HIGHLIM);

    skip_if (0);

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));

    /* Objects observation */
    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);
    rawframes = irplib_framelist_extract_regexp(allframes, "^(" VISIR_IMG_FF_RAW
                                                "|" VISIR_IMG_TECH_FF_RAW
                                                "|" VISIR_SPC_TECH_FF_RAW
                                                "|" VISIR_SPC_FF_RAW ")$",
                                                CPL_FALSE);
    skip_if (rawframes == NULL);
    irplib_framelist_empty(allframes);

    skip_if(irplib_framelist_load_propertylist_all(rawframes, 0,
                                                   visir_property_regexp,
                                                   CPL_FALSE));

    skip_if(visir_dfs_check_framelist_tag(rawframes));

    /* Verify uniqueness of frame type and determine PRO.CATG */
    visir_img_ff_config.img_mode = visir_ff_none;
    if (cpl_frameset_find(framelist, VISIR_IMG_FF_RAW)) {
        visir_img_ff_config.img_mode    = visir_ff_img;
        visir_img_ff_config.procatg_ff  = VISIR_IMG_FF_PROCATG;
        visir_img_ff_config.procatg_bpm = VISIR_IMG_FF_BPM_PROCATG;
    }
    if (cpl_frameset_find(framelist, VISIR_SPC_FF_RAW)) {
        skip_if (visir_img_ff_config.img_mode);
        visir_img_ff_config.img_mode    = visir_ff_spc;
        visir_img_ff_config.procatg_ff  = VISIR_SPC_FF_PROCATG;
        visir_img_ff_config.procatg_bpm = VISIR_SPC_FF_BPM_PROCATG;
    }
    if (cpl_frameset_find(framelist, VISIR_IMG_TECH_FF_RAW)) {
        skip_if (visir_img_ff_config.img_mode);
        visir_img_ff_config.img_mode    = visir_ff_img_tech;
        visir_img_ff_config.procatg_ff  = VISIR_IMG_TECH_FF_PROCATG;
        visir_img_ff_config.procatg_bpm = VISIR_IMG_TECH_FF_BPM_PROCATG;
    }
    if (cpl_frameset_find(framelist, VISIR_SPC_TECH_FF_RAW)) {
        skip_if (visir_img_ff_config.img_mode);
        visir_img_ff_config.img_mode    = visir_ff_spc_tech;
        visir_img_ff_config.procatg_ff  = VISIR_SPC_TECH_FF_PROCATG;
        visir_img_ff_config.procatg_bpm = VISIR_SPC_TECH_FF_BPM_PROCATG;
    }
    
    /* Compute the flatfield  */
    cpl_msg_info(cpl_func, "Compute the flatfield");
    if ((flat = visir_img_ff_reduce(rawframes)) == NULL) {
        cpl_msg_error(cpl_func, "Could not compute the flatfield image");
        skip_if(1);
    }

    /* Compute the bad pixels map */
    cpl_msg_info(cpl_func, "Compute the bad pixels map");
    bpm = cpl_mask_threshold_image_create(cpl_imagelist_get(flat, 0),
                                          visir_img_ff_config.bpm_lo_thresh,
                                          visir_img_ff_config.bpm_hi_thresh);
    if (bpm == NULL) {
        cpl_msg_error(cpl_func, "Could not compute the bad pixel map");
        skip_if(1);
    }
    skip_if (cpl_mask_not(bpm));
    visir_img_ff_config.bpm_nb_bad = cpl_mask_count(bpm);
    bpm_int = cpl_image_new_from_mask(bpm);
    
    /* The values <bpm_lo_thresh and >bpm_hi_thresh are set to 1 */
    skip_if (cpl_image_threshold(cpl_imagelist_get(flat, 0), 
            visir_img_ff_config.bpm_lo_thresh,
            visir_img_ff_config.bpm_hi_thresh, 
            1.0, 1.0));
    
    skip_if (visir_img_ff_qc(qclist, paflist, rawframes));
    irplib_framelist_empty(rawframes);

    skip_if (visir_img_ff_save(framelist, parlist, qclist, paflist, flat,
                               bpm_int));

    end_skip;

    cpl_propertylist_delete(paflist);
    cpl_propertylist_delete(qclist);
    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    cpl_imagelist_delete(flat);
    cpl_mask_delete(bpm);
    cpl_image_delete(bpm_int);

    return cpl_error_get_code();
}

/*----------------------------------------------------------------------------*/
/**
  @brief   Compute the flatfield image
  @param   cplrawframes    the input raw frames
  @return  the images set with the flatfield
 */
/*----------------------------------------------------------------------------*/
static cpl_imagelist * visir_img_ff_reduce(const irplib_framelist * rawframes)
{
    cpl_imagelist * iset   = NULL;
    cpl_imagelist * fitted = NULL;
    cpl_image     * diff   = NULL;
    cpl_image     * prev   = NULL;
    cpl_image     * image  = NULL;
    const cpl_propertylist * plist;
    const char    * sval;


    bug_if (0);
    bug_if (rawframes == NULL);

    plist = irplib_framelist_get_propertylist_const(rawframes, 0);
    bug_if(plist == NULL);

    /* FIXME: Need a separate DO_CATG for lamp flats */
    sval = irplib_pfits_get_dpr_type(plist);
    skip_if (sval == NULL);
    visir_img_ff_config.is_lamp
        = strstr(sval, "SKY") == NULL ? CPL_TRUE : CPL_FALSE;

    /* Load the image set */
    iset = irplib_imagelist_load_framelist(rawframes, CPL_TYPE_FLOAT, 0, 0);
    skip_if (iset == NULL);

    /* Apply the Fit and compute the gain, intercept map and error */
    fitted = irplib_flat_fit_set(iset, 1);

    skip_if (fitted == NULL);

    if (visir_img_ff_config.is_lamp) {
        /* Copied from naco_img_lampflat.c */
        double std_diff[3];
        double upright_x, upright_y;
        int    i;

        upright_x  = cpl_image_get_size_x(cpl_imagelist_get(iset, 0));
        upright_y  = cpl_image_get_size_y(cpl_imagelist_get(iset, 0));
        upright_x -= visir_img_ff_border;
        upright_y -= visir_img_ff_border;
        skip_if(upright_x <= visir_img_ff_border);
        skip_if(upright_y <= visir_img_ff_border);

        for (i=0; i < 4 && i < cpl_imagelist_get_size(iset); i++) {

            double noise;


            cpl_image_delete(prev);
            prev = image;
            image = cpl_image_extract(cpl_imagelist_get(iset, i),
                                      visir_img_ff_border,
                                      visir_img_ff_border,
                                      upright_x,
                                      upright_y);
            bug_if (image == NULL);

            if (i == 0) continue;

            diff = cpl_image_subtract_create(prev, image);
            bug_if (diff == NULL);

            bug_if(cpl_flux_get_noise_window(diff, NULL, -1, -1,
                                             &noise, NULL));

            std_diff[i-1] = noise * noise;

            cpl_msg_debug(cpl_func, "SQ-Noise(%d):     %g", i, std_diff[i-1]);

            if (i == 1) {
                const double median = cpl_image_get_median(diff);
                const double ditval = visir_pfits_get_dit(plist);

                skip_if (0);
                skip_if (ditval <= 0.0);

                visir_img_ff_config.lamp_flux = median / ditval;
                cpl_msg_info(cpl_func, "Lamp Flux: %g",
                             visir_img_ff_config.lamp_flux);
            }

            cpl_image_delete(diff);
            diff = NULL;

        }

        if (i == 4) {

            /* Deduce FPNOISE */
            const double square = std_diff[0] - std_diff[1] - std_diff[2];
            visir_img_ff_config.fp_noise = square > 0.0 ? sqrt(square) : 0.0;

            cpl_msg_info(cpl_func, "Noise:     %g",
                         visir_img_ff_config.fp_noise);

        }
    }

    end_skip;

    cpl_image_delete(diff);
    cpl_image_delete(image);
    cpl_image_delete(prev);
    cpl_imagelist_delete(iset);

    return fitted;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    Append the QC parameters to the propertylist
  @param    qclist    List of QC parameters
  @param    paflist   List of PAF parameters
  @param    rawframes List of rawframes and their propertylists
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_img_ff_qc(cpl_propertylist       * qclist,
                                      cpl_propertylist       * paflist,
                                      const irplib_framelist * rawframes)
{

    const cpl_propertylist * reflist
        = irplib_framelist_get_propertylist_const(rawframes, 0);

    bug_if (0);

    /* QC.CAPA */
    skip_if (visir_qc_append_capa(qclist, rawframes));

    /* QC.NBBADPIX */
    bug_if(cpl_propertylist_append_int(qclist, "ESO QC NBBADPIX", 
                                       visir_img_ff_config.bpm_nb_bad));

    if (visir_img_ff_config.is_lamp) {
        bug_if(cpl_propertylist_append_double(qclist, "ESO QC FPNOISE", 
                                              visir_img_ff_config.fp_noise));
        bug_if(cpl_propertylist_append_double(qclist, "ESO QC LAMPFLUX", 
                                              visir_img_ff_config.lamp_flux));
    }

    bug_if (cpl_propertylist_append(paflist, qclist));

    bug_if (cpl_propertylist_copy_property_regexp(paflist, reflist,
                              (visir_img_ff_config.img_mode & visir_ff_mode_spc)
                               ? PAFCOPY "|" PAF_SPC : PAFCOPY "|" PAF_IMG, 0));

    /* PRO.CATG */
    bug_if (cpl_propertylist_append_string(paflist, CPL_DFS_PRO_CATG,
                                           visir_img_ff_config.procatg_ff));

    bug_if (cpl_propertylist_copy_property_regexp(qclist, reflist,
                                                  "^(" VISIR_PFITS_FF_COPY
                                                  ")$", 0));

    bug_if (irplib_pfits_set_airmass(qclist, rawframes));

    end_skip;
    
    return cpl_error_get_code();

}
 

/*----------------------------------------------------------------------------*/
/**
  @brief    Save the visir_img_ff recipe products on disk
  @param    set         the input frame set
  @param    parlist     the input list of parameters
  @param    qclist      List of QC parameters
  @param    paflist     List of PAF parameters
  @param    flat        the flat images 
  @param    bpm         the bad pixels map
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code visir_img_ff_save(cpl_frameset            * set,
                                        const cpl_parameterlist * parlist, 
                                        const cpl_propertylist  * qclist,
                                        const cpl_propertylist  * paflist,
                                        const cpl_imagelist     * flat,
                                        const cpl_image         * bpm)
{
    /* SAVE THE FLAT IMAGE */
    skip_if (irplib_dfs_save_image(set, parlist, set,
                                cpl_imagelist_get_const(flat, 0),
                                CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
                                visir_img_ff_config.procatg_ff, qclist, NULL,
                                visir_pipe_id, RECIPE_STRING CPL_DFS_FITS));

    /* SAVE THE BPM IMAGE */
    skip_if (irplib_dfs_save_image(set, parlist, set, bpm, CPL_BPP_32_SIGNED,
                                RECIPE_STRING, visir_img_ff_config.procatg_bpm,
                                qclist, NULL, visir_pipe_id,
                                RECIPE_STRING "_bpm" CPL_DFS_FITS));

#ifdef VISIR_SAVE_PAF
    skip_if (cpl_dfs_save_paf("VISIR", RECIPE_STRING, paflist,
                              RECIPE_STRING CPL_DFS_PAF));
#else
    bug_if(paflist == NULL);
#endif

    end_skip;

    return cpl_error_get_code();

}
