/* $Id: visir_util_undistort.c,v 1.30 2013-05-13 16:05:04 jtaylor Exp $
 *
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2002,2003 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

/*
 * $Author: jtaylor $
 * $Date: 2013-05-13 16:05:04 $
 * $Revision: 1.30 $
 * $Name: not supported by cvs2svn $
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
                                Includes
 -----------------------------------------------------------------------------*/

#include "visir_recipe.h"

#include "visir_spc_distortion.h"
#include "visir_cpl_compat.h"
#include "visir_utils.h"
#include <cxlist.h>
#ifdef _OPENMP
#include <omp.h>
#endif

/*-----------------------------------------------------------------------------
                                Defines
 -----------------------------------------------------------------------------*/

#define RECIPE_STRING   "visir_util_undistort"

/*-----------------------------------------------------------------------------
                            Private Functions prototypes
 -----------------------------------------------------------------------------*/

#ifdef VISIR_CHAIN
#define cpl_plugin_get_info visir_util_undistort_get_info
#endif
cpl_recipe_define(visir_util_undistort, VISIR_BINARY_VERSION,
                  "Lars Lundin", PACKAGE_BUGREPORT, "2011", 
                  "Correct the distortion in spectral data",
                  "The files listed in the Set Of Frames (sof-file) "
                  "must be tagged:\n"
                  "VISIR-chopnod-corrected-file.fits " VISIR_UTIL_UNDISTORT_RAW
                  "\nOptionally, a bad pixel map may be provided:\n"
                  "VISIR-bpm-file.fits " VISIR_CALIB_STATIC_MASK "\n"
                  "\nThe product(s) will have a FITS card\n"
                  "'HIERARCH ESO PRO CATG' with a value of:\n"
                  VISIR_UTIL_UNDISTORT_PROCATG "\n"
#ifdef VISIR_UTIL_UNDISTORT_AUTO_REJECT
                  "If no bad pixel map is provided, the recipe will "
                  "automatically flag input intensities greater than or equal "
                  "to 32767 as bad.\n"
#endif
                  "The recipe default values for the transformation are only "
                  "valid for spectral data taken in Low resolution mode");

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_util_undistort   Correct the spectral distortion
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
                                Functions code
 -----------------------------------------------------------------------------*/


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    Fill the recipe parameterlist
  @param    self  The parameterlist
  @return   0 iff everything is ok

 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
visir_util_undistort_fill_parameterlist(cpl_parameterlist * self)
{
    cpl_error_code err;
    const char * context = PACKAGE "." RECIPE_STRING;

    /* --bkgcorrect */
    err = irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
                                        "bkgcorrect", CPL_TRUE, NULL, context,
                                        "Cho-nod correct the data");
    cpl_ensure_code(!err, err);

    /* --xl */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
                                       "xl", 117, NULL, context,
                                       "Coordinate in spatial direction. "
                                       "Together with yl it defines the "
                                       "lower point of a rectangle containing "
                                       "only skylines for the "
                                       "wavelength shift detection");
    cpl_ensure_code(!err, err);

    /* --yl */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
                                       "yl", 110, NULL, context,
                                       "Coordinate in wavelength direction. "
                                       "See xl");
    cpl_ensure_code(!err, err);

    /* --xh */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
                                       "xh", 125, NULL, context,
                                       "Coordinate in spatial direction. "
                                       "Together with yl it defines the "
                                       "higher point of a rectangle containing "
                                       "only skylines for the "
                                       "wavelength shift detection");
    cpl_ensure_code(!err, err);

    /* --yh */
    err = irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
                                       "yh", 150, NULL, context,
                                       "Coordinate in wavelength direction. "
                                       "See xh");
    cpl_ensure_code(!err, err);

    return visir_parameter_set(self, RECIPE_STRING, VISIR_PARAM_SLITSKEW |
                               VISIR_PARAM_SPECSKEW | VISIR_PARAM_VERTARC |
                               VISIR_PARAM_HORIARC)
        ? cpl_error_set_where(cpl_func) : CPL_ERROR_NONE;
}

/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    simple bilinear shift
  @param    out   array to hold output data
  @param    in    input data
  @param    n     array size
  @param    shift the amount to shift the input

  */
/*----------------------------------------------------------------------------*/
static void
shift_data(double * out, const double * in, size_t n, double shift)
{
    /* FIXME: larger smoothing kernel would be better for this purpose */
    if (shift < 0) {
        out[0] = in[0];
        for (size_t i = 1; i < n; i++)
            out[i] = (in[i] - in[i - 1]) * shift + in[i];
    }
    else {
        out[n - 1] = in[n - 1];
        for (size_t i = 0; i < n - 1; i++)
            out[i] = (in[i + 1] - in[i]) * shift + in[i];
    }
}

/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    determine shift in the wavelength (y) direction
  @param    a     base image
  @param    b     shifted image
  @param    xl    image region to look at
  @param    yl    image region to look at
  @param    xh    image region to look at
  @param    yh    image region to look at
  @return   shift of b against a in y direction

  Determines tiny shifts in the wavelength direction (e.g. due to grating
  oscillations)
  The image region to use must contain skylines with amplitudes much larger
  than the signal.
  It first extracts the median in wavelength direction from the image region.
  Then it shifts the base and subtracts the original base to determine
  the shape of the result if there is a shift.
  This result is compared to the on-off subtraction for several shifts.
  The shift where the result is the most similar is chosen.
  */
/*----------------------------------------------------------------------------*/
static double
get_grating_shift(const cpl_image * a, const cpl_image * b,
                  cpl_size xl, cpl_size yl, cpl_size xh, cpl_size yh)
{
    cpl_image * aslice = cpl_image_extract(a, xl, yl, xh, yh);
    cpl_image * bslice = cpl_image_extract(b, xl, yl, xh, yh);
    const size_t n = yh - yl;
    double adata[n];
    double bdata[n];
    cpl_vector * vb;
    cpl_vector * va;
    double minsum = DBL_MAX;
    double best_shift = 0;
    const int step = 5000;
    /* +- shift search range */
    const double range = 1;

    /* extract median of the rows */
    for (size_t i = 0; i < n; i++) {
        va = cpl_vector_new_from_image_row(aslice, i + 1);
        vb = cpl_vector_new_from_image_row(bslice, i + 1);
        adata[i] = cpl_vector_get_median(va);
        bdata[i] = cpl_vector_get_median(vb);
        cpl_vector_delete(va);
        cpl_vector_delete(vb);
    }
    va = cpl_vector_wrap(n, adata);
    vb = cpl_vector_wrap(n, bdata);
    /* correct for intensity offsets */
    cpl_vector_subtract_scalar(va, cpl_vector_get_median_const(va));
    cpl_vector_subtract_scalar(vb, cpl_vector_get_median_const(vb));

    /* real subtraction */
    cpl_vector_subtract(vb, va);

    /* FIXME: can be speed up by doing a coarse grain search first */
    /* brute force search for the best shift */
    for (int i = 0; i < step; i++) {
        const double shift = (double)(i - (step / 2)) / step * range * 2;
        double ashifted[n];
        double sum = 0.;

        shift_data(ashifted, adata, n, shift);

        /* compare squared difference of shifted to real subtraction */
        for (size_t j = 0; j < n; j++) {
            const double tmp = (ashifted[j] - adata[j]) - bdata[j];
            sum += tmp * tmp;
        }

        if (sum < minsum) {
            best_shift = shift;
            minsum = sum;
        }
    }

    cpl_vector_unwrap(va);
    cpl_vector_unwrap(vb);
    cpl_image_delete(aslice);
    cpl_image_delete(bslice);

    cpl_msg_info(cpl_func, "Grating shift: %.3f", best_shift);

    return best_shift;
}

static void
vimglist_append(visir_imglist * a, visir_imglist * b)
{
    for (cpl_size i = 0; i < visir_imglist_get_size(b); i++) {
        visir_imglist_append(a, visir_imglist_get_img(b, i),
                             visir_imglist_get_data(b, i));
    }
    visir_imglist_unwrap(b, NULL);
}

static visir_imglist *
load_images(cpl_frame * frame, const cpl_mask * bpm)
{
    const int next = cpl_frame_get_nextensions(frame);
    const char * filename  = cpl_frame_get_filename(frame);
    cpl_image * img = NULL;
    visir_imglist * res = visir_imglist_new(next, NULL);
    cpl_propertylist * plist = cpl_propertylist_load_regexp(filename, 0,
                                                "ESO DRS CUMOFFSET.",
                                                CPL_FALSE);
    for (int iext = 0; iext < next + 1; iext++) {
        cpl_errorstate prestate = cpl_errorstate_get();

        img = cpl_image_load(filename, CPL_TYPE_FLOAT, 0, iext);
        if (img == NULL) {
            cpl_msg_debug(cpl_func, "No image-data in extension %d", iext);
            cpl_errorstate_set(prestate);
            continue;
        }

        if (bpm != NULL) {
            skip_if(cpl_image_reject_from_mask(img, bpm));
            visir_interpolate_rejected(img, NULL, NULL);
        }

        visir_imglist_append(res, img, cpl_propertylist_duplicate(plist));
    }

    end_skip;

    cpl_propertylist_delete(plist);

    return res;
}


/*----------------------------------------------------------------------------*/
/**
  @brief    use top and bottom unilluminated pixels for overscan bias correction
  @param    img   input image
  @return   0 iff everything is ok

  take row median of unilluminated region and subtract it from complete row
 */
/*----------------------------------------------------------------------------*/
static cpl_error_code
overscan_correct(cpl_image * img)
{
    const cpl_size nx = cpl_image_get_size_x(img);
    const cpl_size ny = cpl_image_get_size_y(img);
    for (cpl_size x = 1; x < nx + 1; x++) {
        /* FIXME: remove hardcoding, maybe use window keys from fits */
        cpl_image * top = cpl_image_extract(img, 1, ny - 7, nx, ny);
        cpl_image * bot = cpl_image_extract(img, 1, 7, nx, 13);
        cpl_vector * t = cpl_vector_new_from_image_column(top, x);
        cpl_vector * b = cpl_vector_new_from_image_column(bot, x);
        double topmed = cpl_vector_get_median(t);
        double botmed = cpl_vector_get_median(b);
        for (cpl_size y = 1; y < ny / 2 + 1; y++) {
            int d;
            cpl_image_set(img, x, y, cpl_image_get(img, x, y, &d) - botmed);
        }
        for (cpl_size y = ny / 2 + 1; y < ny + 1; y++) {
            int d;
            cpl_image_set(img, x, y, cpl_image_get(img, x, y, &d) - topmed);
        }
        cpl_vector_delete(t);
        cpl_vector_delete(b);
        cpl_image_delete(top);
        cpl_image_delete(bot);
    }

    return cpl_error_get_code();
}


static cpl_error_code
handle_images(visir_imglist * list, const cpl_image * base,
              cpl_size xl, cpl_size yl, cpl_size xh, cpl_size yh,
              double phi, double ksi, double eps, double delta,
              const visir_data_type dtype,
              const visir_spc_resol resol)
{
    const cpl_size n = visir_imglist_get_size(list);
    const cpl_size nx = cpl_image_get_size_x(visir_imglist_get_img(list, 0));
    const cpl_size ny = cpl_image_get_size_y(visir_imglist_get_img(list, 0));
    error_if(xh >  nx || yh > ny, CPL_ERROR_ILLEGAL_INPUT,
             "Sky region %d/%d larger than image %d/%d", (int)xh, (int)yh,
             (int)nx, (int)ny);

#ifdef _OPENMP
#pragma omp parallel for
#endif
    for (cpl_size i = 0; i < n; i++) {
        cpl_image * img = visir_imglist_get_img(list, i);

        /* undistort only average for aquarius
         * FIXME: correction required? */
        if (visir_data_is_drs(dtype)) {
            double shift = get_grating_shift(base, img, xl, yl, xh, yh);
            visir_spc_det_warp(&img, 1, 0., shift,  phi, ksi, eps, delta);
        }
        else {
            overscan_correct(img);
            if (resol == VISIR_SPC_R_LRP) {
                cpl_propertylist * plist = visir_imglist_get_data(list, i);
                double dx = cpl_propertylist_get_double(plist,
                                                        VISIR_DRS_CUMOFFSETX);
                double dy = cpl_propertylist_get_double(plist,
                                                        VISIR_DRS_CUMOFFSETY);
                visir_spc_det_warp(&img, 1, -dx, -dy, phi, ksi, eps, delta);
            }
        }
    }

    end_skip;

    return cpl_error_get_code();
}


static cpl_error_code
save_images(visir_imglist * imgs, cpl_frameset * products,
            cpl_frameset * usedframes, const cpl_propertylist * plist,
            const cpl_parameterlist * parlist)
{
    static int j = 0;
    char * proname = cpl_sprintf(RECIPE_STRING "_%03d" CPL_DFS_FITS, j++);

    skip_if(irplib_dfs_save_propertylist(products, parlist, usedframes,
                                         RECIPE_STRING,
                                         VISIR_UTIL_UNDISTORT_PROCATG,
                                         plist, NULL,
                                         visir_pipe_id, proname));
    for (cpl_size i = 0; i < visir_imglist_get_size(imgs); i++) {
        cpl_image * img  = visir_imglist_get_img(imgs, i);
        cpl_image_save(img, proname, CPL_BPP_IEEE_FLOAT, NULL, CPL_IO_EXTEND);
    }

    end_skip;
    cpl_free(proname);

    return cpl_error_get_code();
}

static inline int
comp_expno(cpl_frame * a, cpl_frame * b)
{
    const char * fna = cpl_frame_get_filename(a);
    const char * fnb = cpl_frame_get_filename(b);
    cpl_propertylist * alist = cpl_propertylist_load(fna, 0);
    cpl_propertylist * blist = cpl_propertylist_load(fnb, 0);

    int expa = cpl_propertylist_get_int(alist, "ESO TPL EXPNO");
    int expb = cpl_propertylist_get_int(blist, "ESO TPL EXPNO");
    int ret = 0;

    if (expa < expb)
        ret = -1;
    if (expa > expb)
        ret = 1;

    cpl_propertylist_delete(alist);
    cpl_propertylist_delete(blist);
    return ret;
}

static cx_list *
sort_framelist(irplib_framelist * allframes, const char * tag)
{
    irplib_framelist * on =
        irplib_framelist_extract_regexp(allframes, tag, CPL_FALSE);
    cx_list * frames = cx_list_new();

    for (int i = irplib_framelist_get_size(on); i-- > 0;)
        cx_list_push_back(frames, irplib_framelist_unset(on, i, NULL));

    /* FIXME: optimize */
    cx_list_sort(frames, (cx_compare_func)comp_expno);

    irplib_framelist_delete(on);

    return frames;
}


cpl_error_code
check_rawframes(const irplib_framelist * rawframes,
                cpl_propertylist * plist)
{
    cpl_errorstate cleanstate = cpl_errorstate_get();
    const char * dit_key = VISIR_PFITS_DOUBLE_DIT;
    if (cpl_propertylist_has(plist, VISIR_PFITS_DOUBLE_SEQ1_DIT))
        dit_key = VISIR_PFITS_DOUBLE_SEQ1_DIT;

    /* DIT must be present every where */
    skip_if(irplib_framelist_contains(rawframes, dit_key,
                                      CPL_TYPE_DOUBLE, CPL_FALSE, 0.0));
    if(irplib_framelist_contains(rawframes, dit_key,
                                 CPL_TYPE_DOUBLE, CPL_TRUE, 1e-5)) {
        /* Allow 0.1 ms difference - or warn */
        /* FIXME: The recipe does not properly handle non-uniform DITSs */
        visir_error_reset("DIT differs by more than %g", 1e-5);
    }

    /* FIXME: Verify the angular distance */
    skip_if(irplib_framelist_contains(rawframes, VISIR_PFITS_DOUBLE_RA,
                                      CPL_TYPE_DOUBLE, CPL_FALSE, 0.0));

    /* FIXME: Allow 1 degree difference */
    skip_if(irplib_framelist_contains(rawframes, VISIR_PFITS_DOUBLE_DEC,
                                      CPL_TYPE_DOUBLE, CPL_TRUE, 1.0));

    skip_if(irplib_framelist_contains(rawframes, VISIR_PFITS_INT_CHOP_NCYCLES,
                                      CPL_TYPE_INT, CPL_TRUE, 0.0));

    skip_if(irplib_framelist_contains(rawframes, VISIR_PFITS_INT_NDIT,
                                      CPL_TYPE_INT, CPL_TRUE, 0.0));

    if (irplib_framelist_contains(rawframes, VISIR_PFITS_STRING_STARNAME,
                                  CPL_TYPE_STRING, CPL_TRUE, 0.0)) {
        visir_error_reset("Rawframe(s) missing standard star name");
    }

    int nnod = irplib_framelist_get_size(rawframes);
    /* frames not chop corrected, so divide the chop factor away again */
    double exptime = visir_utils_get_exptime(nnod, plist) / 2.;
    skip_if(0);

    cpl_propertylist_append_double(plist, "ESO QC EXPTIME", exptime);

    end_skip;

    return cpl_error_get_code();
}


static void update_exectime(const cpl_frame * frm, double * t_min_obsstart,
                            double * t_max_filewrite)
{
    cpl_propertylist * plist = cpl_propertylist_load(cpl_frame_get_filename(frm), 0);
    if (plist == NULL || !cpl_propertylist_has(plist, "ESO DRS DATE") ||
        !cpl_propertylist_has(plist, "ESO DRS DATE-OBS")) {
        cpl_propertylist_delete(plist);
        return;
    }
    *t_max_filewrite =
        CX_MAX(cpl_propertylist_get_double(plist, "ESO DRS DATE"),
               *t_max_filewrite);
    *t_min_obsstart =
        CX_MIN(cpl_propertylist_get_double(plist, "ESO DRS DATE-OBS"),
               *t_min_obsstart);
    cpl_propertylist_delete(plist);
}

/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here 
  @param    framelist   the frames list
  @param    parlist     the parameters list
  @return   0 iff everything is ok
 */
/*----------------------------------------------------------------------------*/
static int visir_util_undistort(cpl_frameset            * framelist,
                                const cpl_parameterlist * parlist)
{
    cpl_errorstate     cleanstate = cpl_errorstate_get();
    const cpl_frame  * bpmframe   = cpl_frameset_find_const
        (framelist, VISIR_CALIB_STATIC_MASK);
    irplib_framelist * allframes  = NULL;
    irplib_framelist * rawframes  = NULL;
    irplib_framelist * rawframes_a = NULL;
    irplib_framelist * rawframes_b = NULL;
    cpl_image        * imbpm      = NULL;
    cpl_mask         * bpm        = NULL;

#ifdef _OPENMP
    omp_set_num_threads(visir_get_num_threads(CPL_FALSE));
#endif

    const cpl_boolean bkgcorrect =
        irplib_parameterlist_get_bool(parlist, PACKAGE,
                                      RECIPE_STRING, "bkgcorrect");

    /* The angles are given in degrees and then converted to radians */

    double ksi   = visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                                        VISIR_PARAM_SPECSKEW)
        * CPL_MATH_RAD_DEG;
    double eps   = visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                                        VISIR_PARAM_VERTARC);
    double delta = visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                                        VISIR_PARAM_HORIARC);
    double phi   = visir_parameterlist_get_double(parlist, RECIPE_STRING,
                                                        VISIR_PARAM_SLITSKEW)
        * CPL_MATH_RAD_DEG;

    const cpl_size xl = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                     RECIPE_STRING, "xl");
    const cpl_size yl = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                     RECIPE_STRING, "yl");
    const cpl_size xh = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                     RECIPE_STRING, "xh");
    const cpl_size yh = irplib_parameterlist_get_int(parlist, PACKAGE,
                                                     RECIPE_STRING, "yh");

    visir_imglist * aon  = visir_imglist_new(100, NULL);
    visir_imglist * aoff = visir_imglist_new(100, NULL);
    visir_imglist * bon  = visir_imglist_new(100, NULL);
    visir_imglist * boff = visir_imglist_new(100, NULL);

    cpl_frameset * usedframes = cpl_frameset_new();
    cpl_image * base = NULL;
    cpl_propertylist * plist = NULL;
    visir_spc_resol resol;
    visir_data_type data_type;
    double wlen;

    /* Identify the RAW and CALIB frames in the input frameset */
    skip_if (visir_dfs_set_groups(framelist));

    error_if(xl < 1 || yl < 1, CPL_ERROR_ILLEGAL_INPUT,
             "Sky region %d/%d must be larger than 1", (int)xl, (int)yl);

    /* Objects observation */
    allframes = irplib_framelist_cast(framelist);
    skip_if(allframes == NULL);

    rawframes = irplib_framelist_extract_regexp(allframes, "^("
                                                VISIR_UTIL_REPACK_A_ON_PROCATG "|"
                                                VISIR_UTIL_REPACK_A_OFF_PROCATG "|"
                                                VISIR_UTIL_REPACK_B_ON_PROCATG "|"
                                                VISIR_UTIL_REPACK_B_OFF_PROCATG
                                                ")$", CPL_FALSE);
    skip_if(irplib_framelist_load_propertylist_all(rawframes, 0, ".*", 0));
    plist = irplib_framelist_get_propertylist(rawframes, 0);
    skip_if(check_rawframes(rawframes, plist));

    {
        double pslitw, ptemp, pfwhm;
        const cpl_frame * frm = irplib_framelist_get_const(rawframes, 0);
        skip_if(visir_get_data_type(frm, plist, &data_type, NULL));
        resol = visir_spc_get_res_wl(rawframes, &wlen, &pslitw, &ptemp, &pfwhm,
                                     visir_data_is_aqu(data_type));
        skip_if(0);
    }

    if (visir_data_is_drs(data_type)) {
        /* if parameters are (aqu) defaults and drs data use old parameters */
        const cpl_parameter * par =
            cpl_parameterlist_find_const(parlist, PACKAGE "." RECIPE_STRING ".ksi");
        if (ksi == cpl_parameter_get_default_double(par) * CPL_MATH_RAD_DEG) {
            ksi = VISIR_DRS_DIST_KSI;
        }
        par = cpl_parameterlist_find_const(parlist, PACKAGE "." RECIPE_STRING ".eps");
        if (eps == cpl_parameter_get_default_double(par)) {
            eps = VISIR_DRS_DIST_EPS;
        }
        par = cpl_parameterlist_find_const(parlist, PACKAGE "." RECIPE_STRING ".delta");
        if (delta == cpl_parameter_get_default_double(par)) {
            delta = VISIR_DRS_DIST_DELTA;
        }
        par = cpl_parameterlist_find_const(parlist, PACKAGE "." RECIPE_STRING ".phi");
        if (phi == cpl_parameter_get_default_double(par) * CPL_MATH_RAD_DEG) {
            phi = VISIR_DRS_DIST_PHI;
        }
    }

    rawframes_a = irplib_framelist_extract_regexp(allframes, "^("
                                                  VISIR_UTIL_REPACK_A_ON_PROCATG "|"
                                                  VISIR_UTIL_REPACK_A_OFF_PROCATG
                                                  ")$", CPL_FALSE);
    cpl_errorstate_set(cleanstate);

    rawframes_b = irplib_framelist_extract_regexp(allframes, "^("
                                                  VISIR_UTIL_REPACK_B_ON_PROCATG "|"
                                                  VISIR_UTIL_REPACK_B_OFF_PROCATG
                                                  ")$", CPL_FALSE);
    cpl_errorstate_set(cleanstate);

    error_if (rawframes_a == NULL && rawframes_b == NULL,
              CPL_ERROR_DATA_NOT_FOUND, "No frames with tags "
              VISIR_UTIL_REPACK_A_ON_PROCATG "|"
              VISIR_UTIL_REPACK_A_OFF_PROCATG "|"
              VISIR_UTIL_REPACK_B_ON_PROCATG "|"
              VISIR_UTIL_REPACK_B_OFF_PROCATG " found");

    if (bpmframe != NULL) {
        /* Load any static mask here to avoid that the file is loaded
           multiple times */
        const char * badpix = cpl_frame_get_filename(bpmframe);


        bpm = cpl_mask_load(badpix, 0, 0);
        skip_if (bpm == NULL);
    }
    double t_min_obsstart = 1e300;
    double t_max_filewrite = 0;
    
    if (rawframes_a) {
        cx_list * on_frames = sort_framelist(rawframes_a, "^("
                                             VISIR_UTIL_REPACK_A_ON_PROCATG
                                             ")$");
        cx_list * off_frames = sort_framelist(rawframes_a, "^("
                                              VISIR_UTIL_REPACK_A_OFF_PROCATG
                                              ")$");
        FOR_EACH_T(cpl_frame * frm, on_frames) {
            update_exectime(frm, &t_min_obsstart, &t_max_filewrite);
            vimglist_append(aon, load_images(frm, bpm));
            cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
            cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
        }
        cx_list_destroy(on_frames, (visir_free)cpl_frame_delete);
        FOR_EACH_T(cpl_frame * frm, off_frames) {
            update_exectime(frm, &t_min_obsstart, &t_max_filewrite);
            vimglist_append(aoff, load_images(frm, bpm));
            cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
            cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
        }
        cx_list_destroy(off_frames, (visir_free)cpl_frame_delete);

        base = cpl_image_duplicate(visir_imglist_get_img(aoff, 0));
        if (visir_data_is_aqu(data_type))
            overscan_correct(base);
        skip_if(handle_images(aon, base, xl, yl, xh, yh,
                              phi, ksi, eps, delta, data_type, resol));
        skip_if(handle_images(aoff, base, xl, yl, xh, yh,
                              phi, ksi, eps, delta, data_type, resol));
    }
    if (rawframes_b){
        cx_list * on_frames = sort_framelist(rawframes_b, "^("
                                             VISIR_UTIL_REPACK_B_ON_PROCATG
                                             ")$");
        cx_list * off_frames = sort_framelist(rawframes_b, "^("
                                              VISIR_UTIL_REPACK_B_OFF_PROCATG
                                              ")$");
        FOR_EACH_T(cpl_frame * frm, on_frames) {
            update_exectime(frm, &t_min_obsstart, &t_max_filewrite);
            vimglist_append(bon, load_images(frm, bpm));
            cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
            cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
        }
        cx_list_destroy(on_frames, (visir_free)cpl_frame_delete);
        FOR_EACH_T(cpl_frame * frm, off_frames) {
            update_exectime(frm, &t_min_obsstart, &t_max_filewrite);
            vimglist_append(boff, load_images(frm, bpm));
            cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
            cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
        }
        cx_list_destroy(off_frames, (visir_free)cpl_frame_delete);

        if (base == NULL) {
            base = cpl_image_duplicate(visir_imglist_get_img(boff, 0));
            if (visir_data_is_aqu(data_type))
                overscan_correct(base);
        }

        skip_if(handle_images(bon, base, xl, yl, xh, yh,
                              phi, ksi, eps, delta, data_type, resol));
        skip_if(handle_images(boff, base, xl, yl, xh, yh,
                              phi, ksi, eps, delta, data_type, resol));
    }

    if (bkgcorrect && rawframes_a && rawframes_b) {
        cpl_imagelist * laon = visir_imglist_get_imglist(aon);
        cpl_imagelist * laoff = visir_imglist_get_imglist(aoff);
        cpl_imagelist_subtract(laon, laoff);

        cpl_imagelist * lbon = visir_imglist_get_imglist(bon);
        cpl_imagelist * lboff = visir_imglist_get_imglist(boff);
        cpl_imagelist_subtract(lbon, lboff);

        cpl_imagelist_subtract(laon, lbon);

        visir_imglist * l = visir_imglist_new(100, NULL);

        {
            cpl_image * img = cpl_imagelist_collapse_create(laon);
            cpl_image * flipped = visir_spc_flip(img, wlen, resol, data_type);
            cpl_image_delete(img);
            img = flipped;

            visir_imglist_append(l, img, NULL);
        }

        cpl_propertylist_append_double(plist, "ESO QC EXECTIME",
                           (t_max_filewrite - t_min_obsstart) * 24 * 3600);
        save_images(l, framelist, usedframes, plist, parlist);
        visir_imglist_delete(l, (visir_free)cpl_propertylist_delete);
        skip_if(0);
    }
    else {
        bug_if(bkgcorrect); /* FIXME */
        save_images(aon, framelist, usedframes, plist, parlist);
        save_images(aoff, framelist, usedframes, plist, parlist);
        save_images(bon, framelist, usedframes, plist, parlist);
        save_images(boff, framelist, usedframes, plist, parlist);
    }

    {
        cpl_frame * frm = cpl_frameset_find(framelist, VISIR_CALIB_LIN);
        if (visir_data_is_aqu(data_type) && frm) {
            cpl_msg_info(cpl_func, "Correcting linearity of skyframe");
            cpl_frameset_insert(usedframes, cpl_frame_duplicate(frm));
            cpl_bivector * lintable = visir_load_lintable(frm, CPL_TRUE);
            if (lintable) {
                cpl_image * dvals = cpl_image_cast(base, CPL_TYPE_DOUBLE);
                /* ugly, revert normalization done by repack */
                cpl_image_multiply_scalar(dvals, visir_pfits_get_dit(plist));
                cpl_image * corvals = visir_linintp_values(dvals, lintable);
                cpl_image_divide(base, corvals);
                cpl_image_delete(dvals);
                cpl_image_delete(corvals);
            }
        }

        skip_if(visir_spc_det_warp(&base, 1, 0., 0.,  phi, ksi, eps, delta));
        cpl_image * flipped = visir_spc_flip(base, wlen, resol, data_type);
        cpl_image_delete(base);
        base = flipped;
    }
    skip_if(irplib_dfs_save_image(framelist, parlist, usedframes,
                                  base, CPL_TYPE_UNSPECIFIED,
                                  RECIPE_STRING,
                                  "SPEC_OBS_LMR_SKYFRAME", NULL, NULL,
                                  visir_pipe_id, "skyframe.fits"));

    end_skip;

    irplib_framelist_delete(allframes);
    irplib_framelist_delete(rawframes);
    irplib_framelist_delete(rawframes_a);
    irplib_framelist_delete(rawframes_b);
    cpl_image_delete(imbpm);
    cpl_mask_delete(bpm);
    cpl_image_delete(base);
    visir_imglist_delete(aon, (visir_free)&cpl_propertylist_delete);
    visir_imglist_delete(aoff, (visir_free)&cpl_propertylist_delete);
    visir_imglist_delete(bon, (visir_free)&cpl_propertylist_delete);
    visir_imglist_delete(boff, (visir_free)&cpl_propertylist_delete);
    cpl_frameset_delete(usedframes);

    return cpl_error_get_code();
}
