/*
 * This file is part of the VISIR Pipeline
 * Copyright (C) 2012,2013,2014 European Southern Observatory
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02111-1307  USA
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

/*-----------------------------------------------------------------------------
  Includes
  -----------------------------------------------------------------------------*/

#include "visir_recipe.h"
#include "visir_utils.h"
#include "irplib_framelist.h"
#include <cxlist.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/types.h>
#include <sys/wait.h>
/* for access */
#include <unistd.h>

#include <fcntl.h>
#include <semaphore.h>

/*-----------------------------------------------------------------------------
  Defines
  -----------------------------------------------------------------------------*/

#define RECIPE_STRING   "visir_util_run_swarp"
#define DEFAULT_CONFIG  VISIR_CONFIG_PATH "/visir_default.swarp"
#define SWARP_COADD_HEAD RECIPE_STRING "_coadd.head"
#define SWARP_DATA_FILES RECIPE_STRING "_data_files"

/*-----------------------------------------------------------------------------
  Private Functions prototypes
  -----------------------------------------------------------------------------*/

#ifdef VISIR_CHAIN
#define cpl_plugin_get_info visir_util_run_swarp_get_info
#endif
cpl_recipe_define(visir_util_run_swarp, VISIR_BINARY_VERSION,
                  "Julian Taylor", PACKAGE_BUGREPORT, "2012",
                  "Run swarp program to coadd images.",
                  "WARNING: this recipe is intented to be run in a "
                  "temporary directory, it does not clean up behind itself\n"
                  "The files listed in the Set Of Frames (sof-file) "
                  "must be tagged:\n"
                  "VISIR-chopnod-corrected-file.fits "VISIR_UTIL_CORRECTED"\n"
                  "VISIR-error-map.fits "VISIR_UTIL_ERROR_MAP
                  "\nIt will produce the coaddition of each input frame "
                  "separately and of all frames together.\n"
                  "The single frame product(s) will have a FITS card\n"
                  "'HIERARCH ESO PRO CATG' with a value of: COADDED_IMAGE and "
                  "COADDED_WEIGHT\nThe total coaddition will have the tags "
                  VISIR_IMG_COADDED_IMG" and "VISIR_IMG_COADDED_WGT);

/*----------------------------------------------------------------------------*/
/**
 * @defgroup visir_util_run_swarp   remove stripes
 */
/*----------------------------------------------------------------------------*/

/*-----------------------------------------------------------------------------
  Functions code
  -----------------------------------------------------------------------------*/

typedef enum {
    VISIR_AVG_COMB,
    VISIR_SUM_COMB
} visir_comb_type;


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    Fill the recipe parameterlist
  @param    self  The parameterlist
  @return   0 iff everything is ok

*/
/*----------------------------------------------------------------------------*/
static cpl_error_code
visir_util_run_swarp_fill_parameterlist(cpl_parameterlist * self)
{
    const char * context = PACKAGE ".visir_util_run_swarp";

    skip_if(irplib_parameterlist_set_bool(self, PACKAGE, RECIPE_STRING,
                                          "output-all", CPL_FALSE, NULL,
                                          context, "Output a coadded image "
                                          "for each input file in addition "
                                          "to the complete coaddition."));

    skip_if(irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
                                            "extra_config", "NONE", NULL, context,
                                            "Additional configuration parameters"));

    skip_if(irplib_parameterlist_set_string(self, PACKAGE, RECIPE_STRING,
                                            "config_fname", DEFAULT_CONFIG, NULL,
                                            context, "Swarp configure file name."));

    skip_if(irplib_parameterlist_set_int(self, PACKAGE, RECIPE_STRING,
                                         "nprocs", -1, NULL, context,
                                         "Maximum number of swarp processes "
                                         "that can be spawned simultaneously."));
    end_skip;

    return cpl_error_get_code();
}


static cpl_error_code
check_swarp(void)
{
    int ret = system(VISIR_SWARP_BIN " -v > /dev/null 2>/dev/null");
    if (WEXITSTATUS(ret) != 0) {
        cpl_error_set_message(cpl_func, CPL_ERROR_UNSUPPORTED_MODE,
                              "swarp not found in PATH");
        return CPL_ERROR_UNSUPPORTED_MODE;
    }
    return CPL_ERROR_NONE;
}


static cx_list *
sort_framelist(const cpl_frameset * allframes, const char * tag)
{
    cx_list * frames = cx_list_new();

    FOR_EACH_FRAMESET_C(frm, allframes) {
        if (tag == NULL || strcmp(cpl_frame_get_tag(frm), tag) == 0)
            cx_list_push_back(frames, frm);
    }

    cx_list_sort(frames, (cx_compare_func)visir_cmp_frm_fn);

    return frames;
}


static cpl_error_code
create_swarp_frames(cx_list * img_list, cx_list * err_list,
                    const cpl_frameset * frameset)
{
    cx_list * imgframes = sort_framelist(frameset, VISIR_UTIL_CORRECTED);
    cx_list * errframes = sort_framelist(frameset, VISIR_UTIL_ERROR_MAP);
    skip_if(0);

    FOR_EACH_T(cpl_frame * frm, imgframes) {
        cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
        cx_list_push_back(img_list, cpl_frame_duplicate(frm));
    }

    FOR_EACH_T(cpl_frame * frm, errframes) {
        cpl_frame_set_group(frm, CPL_FRAME_GROUP_RAW);
        cx_list_push_back(err_list, cpl_frame_duplicate(frm));
    }

    /* FIXME: check #ext of frames */
    error_if(cx_list_size(err_list) != 0 &&
             cx_list_size(img_list) != cx_list_size(err_list),
             CPL_ERROR_UNSUPPORTED_MODE,
             "Number of error frames does not match image frames");

    end_skip;

    cx_list_delete(imgframes);
    cx_list_delete(errframes);

    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    coadd a single frame by forking swarp into the background
  @param    imgfrm  frame with images
  @param    errfrm  frame with errors
  @return   pids of forked process

  forks swarp into background which coadds a frame
  swarp has quite aweful multithreaded performance so forking multiple
  processes is faster.
  One can fork a swarp per file when a coadd.head defining the output
  projection of all files is present in the working directory.
  The output files can then be simply added without further shifting.
*/
/*----------------------------------------------------------------------------*/
static pid_t
run_swarp_resample(cpl_frame * imgfrm, cpl_frame * errfrm,
                   const cpl_parameterlist * parlist,
                   char * basedir,
                   int ext_low, int ext_high,
                   sem_t * sem_resamp)
{
    cpl_msg_debug(cpl_func, "start swarp %s: %d %d", cpl_frame_get_filename(imgfrm), ext_low, ext_high);
    pid_t pid = fork();
    if (pid == 0) {
        char * cmd;
        char datafile[128];
        char errorfile[128];
        const char * config =
            irplib_parameterlist_get_string(parlist, PACKAGE,
                                            RECIPE_STRING, "config_fname");
        const char * extra =
            irplib_parameterlist_get_string(parlist, PACKAGE,
                                            RECIPE_STRING, "extra_config");
        if (strcasecmp(extra, "NONE") == 0)
            extra = "";
        pid_t cpid = getpid();
        sprintf(datafile, "swarp_data_%d.list", (int)cpid);
        sprintf(errorfile, "swarp_error_%d.list", (int)cpid);

        FILE * data = fopen(datafile, "w");
        if (cpl_frame_get_filename(imgfrm)[0] != '/') {
            fprintf(data, "%s/%s[%d:%d]\n", basedir,
                    cpl_frame_get_filename(imgfrm), ext_low, ext_high);
        }
        else {
            fprintf(data, "%s[%d:%d]\n", cpl_frame_get_filename(imgfrm),
                    ext_low, ext_high);
        }
        fclose(data);

        FILE * error = fopen(errorfile, "w");
        if (cpl_frame_get_filename(errfrm)[0] != '/') {
            fprintf(error, "%s/%s[%d:%d]\n", basedir,
                    cpl_frame_get_filename(errfrm), ext_low, ext_high);
        }
        else {
            fprintf(error, "%s[%d:%d]\n", cpl_frame_get_filename(errfrm),
                    ext_low, ext_high);
        }
        fclose(error);

        cmd = cpl_sprintf(VISIR_SWARP_BIN " -c %s @%s -NTHREADS 1 "
                          "-BLANK_BADPIXELS Y "
                          "-WEIGHT_IMAGE @%s "
                          "-WEIGHT_TYPE MAP_RMS %s "
                          "-COMBINE N "
                          "> \"%s/swarp-resamp_%d.log\" 2>&1",
                          config, datafile, errorfile, extra, basedir, (int)cpid);
        cpl_msg_debug(cpl_func, "RESAMPLE: %s", cmd);
        int status = system(cmd);
        cpl_free(cmd);

        /* notify parent that a resample has ended */
        sem_post(sem_resamp);
        /* close our forks copy of the semaphore */
        sem_close(sem_resamp);
        _exit(WEXITSTATUS(status));
    }
    else {
        return pid;
    }
}

/* ---------------------------------------------------------------------------*/
/**
 * @brief resample and coadd one frame in a fork
 *
 * @param imgfrm      image frame
 * @param errfrm      error frame
 * @param parlist     parameter list
 * @param sem_coadd   coaddition semaphore, limited to 1
 * @param sem_resamp  resampling semaphore, limited to number of processors
 *
 * resample and coadd one frame into a common projection. Will start multiple
 * parallel resampling steps and allows further resample-coaddition steps to
 * start of more resampling jobs but will only allow another coadding job when
 * it is done.
 */
/* ---------------------------------------------------------------------------*/
static pid_t
run_swarp_coadd(cpl_frame * imgfrm, cpl_frame * errfrm,
                const cpl_parameterlist * parlist,
                sem_t * sem_coadd, sem_t * sem_resamp)
{
    pid_t pid = fork();
    if (pid == 0) {
        /* start multiple resampling processes on slices of the extensions of
         * the frame */
        cpl_size next = cpl_frame_get_nextensions(imgfrm);
        cpl_size bs = 300;
        pid_t lpids[next / bs + ((next % bs) != 0)];
        int l = 0;
        char tmpdir[] = RECIPE_STRING"_XXXXXX";
	char * basedir = visir_get_cwd();
        if (basedir == NULL ||
            visir_get_tempdir(tmpdir) != CPL_TRUE || chdir(tmpdir) != 0) {
            cpl_msg_error(cpl_func, "Creating temporary directory failed");
            /* notify parent that coadd job failed */
            sem_post(sem_coadd);
            /* close our forks copy of the semaphores */
            sem_close(sem_coadd);
            sem_close(sem_resamp);
	    cpl_free(basedir);
            _exit(EXIT_FAILURE);
        }

        /* duplicate the header defining the final projection
         * to each output file */
        char * cmd = cpl_sprintf("cp \"%s/"SWARP_COADD_HEAD"\" coadd.head", basedir);
        int status = system(cmd);
        cpl_free(cmd);
        if (WEXITSTATUS(status) != 0) {
            cpl_msg_error(cpl_func, "copy of coaddition header failed");
            /* notify parent that coadd job failed */
            sem_post(sem_coadd);
            /* close our forks copy of the semaphores */
            sem_close(sem_coadd);
            sem_close(sem_resamp);
	    cpl_free(basedir);
            _exit(EXIT_FAILURE);
        }

        for (cpl_size b = 1; b < next + 1; b += bs) {
            /* wait to not start too many resamplings */
            sem_wait(sem_resamp);
            pid_t resamp_pid = run_swarp_resample(imgfrm, errfrm, parlist,
                                                  basedir, b,
                                                  CX_MIN(b + bs, next + 1),
                                                  sem_resamp);
            lpids[l++] = resamp_pid;
        }
        /* notify parent that the next resample->coadd can start, but the
         * sem_resamp blocks them until enough of this coadd are done */
        sem_post(sem_coadd);
        /* close our forks copy of the semaphores */
        sem_close(sem_coadd);
        sem_close(sem_resamp);

        for (int k = 0; k < l; k++) {
            pid_t r = waitpid(lpids[k], &status, 0);
            if (r == -1 || !WIFEXITED(status) || WEXITSTATUS(status))
                _exit(WEXITSTATUS(status));
            cpl_msg_debug(cpl_func, "resamp %s %d done",
                          cpl_frame_get_filename(imgfrm), k);
        }

        const char * config =
            irplib_parameterlist_get_string(parlist, PACKAGE,
                                            RECIPE_STRING, "config_fname");
        const char * extra =
            irplib_parameterlist_get_string(parlist, PACKAGE,
                                            RECIPE_STRING, "extra_config");
        if (strcasecmp(extra, "NONE") == 0)
            extra = "";
        pid_t cpid = getpid();
        char * coadd_img = cpl_sprintf("%s/tmp_coadd_%d.fits",
                                       basedir, (int)cpid);
        char * coadd_weight = cpl_sprintf("%s/tmp_coadd_wgt_%d.fits",
                                          basedir, (int)cpid);

        /* duplicate the header defining the final projection
         * to each output file , searches for IMAGEOUT_NAME.head */
        cmd = cpl_sprintf("cp \"%s/"SWARP_COADD_HEAD"\" \"%s/tmp_coadd_%d.head\"",
                          basedir, basedir, (int)cpid);
        status = system(cmd);
        cpl_free(cmd);
        if (WEXITSTATUS(status) != 0) {
            cpl_msg_error(cpl_func, "copy of coaddition header failed");
            cpl_free(coadd_img);
            cpl_free(coadd_weight);
	    cpl_free(basedir);
            _exit(WEXITSTATUS(status));
        }

        cmd = cpl_sprintf(VISIR_SWARP_BIN" -c %s $(ls *resamp.fits) "
                          "-NTHREADS 1 -BLANK_BADPIXELS Y "
                          "-WEIGHT_TYPE MAP_WEIGHT %s "
                          "-IMAGEOUT_NAME %s "
                          "-WEIGHTOUT_NAME %s "
                          "-RESAMPLE N -COMBINE Y "
                          "> %s/swarp-coadd-%d.log 2>&1",
                          config, extra,
                          coadd_img, coadd_weight, basedir, (int)cpid);
        cpl_msg_debug(cpl_func, "COADD: %s", cmd);
        status = system(cmd);
        cpl_free(cmd);
        if (chdir(basedir) == 0) {
            cmd = cpl_sprintf("rm -rf \"%s\"", tmpdir);
            status = system(cmd);
            cpl_free(cmd);
        }
        cpl_free(coadd_img);
        cpl_free(coadd_weight);

        _exit(WEXITSTATUS(status));
    }
    else {
        return pid;
    }
}


static cpl_error_code
generate_coaddition_header(const cx_list * imglist, const char * config, const char * extra)
{
    /* generate coaddition header which is read by each coaddition
     * process to get the correct final output projection */
    FILE * data = fopen(SWARP_DATA_FILES, "w");
    char * cmd = cpl_sprintf(VISIR_SWARP_BIN " -c %s @"SWARP_DATA_FILES
                             " -IMAGEOUT_NAME "SWARP_COADD_HEAD
                             " -HEADER_ONLY Y %s > /dev/null 2>/dev/null",
                             config, extra);
    int status;

    FOR_EACH_T(cpl_frame * frm, imglist) {
        int ret;
        cpl_propertylist * plist =
            cpl_propertylist_load(cpl_frame_get_filename(frm), 0);
        cpl_errorstate cleanstate = cpl_errorstate_get();
        cpl_boolean invalid =
            (cpl_propertylist_get_double(plist, "CD1_1") == 0. &&
             cpl_propertylist_get_double(plist, "CD1_2") == 0.) ||
            (cpl_propertylist_get_double(plist, "CD2_1") == 0. &&
             cpl_propertylist_get_double(plist, "CD2_2") == 0.);
        invalid = invalid || (cpl_error_get_code() != CPL_ERROR_NONE);
        cpl_errorstate_set(cleanstate);
        cpl_propertylist_delete(plist);
        error_if (invalid, CPL_ERROR_ILLEGAL_INPUT,
                  "CDX_Y WCS key missing or zero");

        cpl_msg_info(cpl_func, "%s",  cpl_frame_get_filename(frm));
        ret = fprintf(data, "%s\n", cpl_frame_get_filename(frm));
        error_if(ret < 0, CPL_ERROR_FILE_IO, "Failed writting swarp file list");
    }
    fclose(data);

    cpl_msg_info(cpl_func, "Generating coaddition header");
    status = system(cmd);

    error_if(WEXITSTATUS(status) != 0, CPL_ERROR_FILE_IO, "Swarp failed");

    end_skip;
    cpl_free(cmd);

    return cpl_error_get_code();
}


static cpl_error_code
coadd_images(pid_t * pids, unsigned long * beamids, float * imgwgts,
             cpl_frameset * usedframes,
             const cx_list * imglist, const cx_list * errlist,
             const int nprocs, const cpl_parameterlist * parlist)
{
    const int nfiles = cx_list_size(imglist);
    int i = 0;
    /* setup tiered load balancing semaphores
     * start multiple coadding jobs at once
     * coadding jobs launch resampling jobs until the resampling semaphore blocks
     * the coadding semaphore blocks at one but is incremented when all
     * resampling jobs are posted so the other coadd jobs can queue in their
     * resampling jobs.
     * That way we are always resampling but only one IO bound coaddition is
     * running at a time as long as the resampling queue is full. */
    char * sem_resamp_name = cpl_sprintf("/visir_swarp_resamp_%d_%d",
                                         getuid(), getpid());
    char * sem_coadd_name = cpl_sprintf("/visir_swarp_coadd_%d_%d",
                                        getuid(), getpid());
    sem_t * sem_resamp = sem_open(sem_resamp_name , O_CREAT, 0644, nprocs);
    sem_t * sem_coadd = sem_open(sem_coadd_name , O_CREAT, 0644, 1);
    sem_unlink(sem_coadd_name);
    sem_unlink(sem_resamp_name);

    for(cx_list_iterator iit = cx_list_begin(imglist),
        eit = cx_list_begin(errlist);
        iit != cx_list_end(imglist);
        iit = cx_list_next(imglist, iit),
        eit = cx_list_next(errlist, eit)) {
        const char * beamid;
        cpl_frame * ifrm = cx_list_get(imglist, iit);
        cpl_frame * efrm = cx_list_get(errlist, eit);
        cpl_propertylist * plist =
            cpl_propertylist_load(cpl_frame_get_filename(ifrm), 0);
        skip_if(0);

        beamid = cpl_propertylist_get_string(plist, "ESO QC BEAMID");
        beamids[i] = strtoul(beamid, NULL, 10);
        imgwgts[i] = visir_pfits_get_img_weight(plist);
        cpl_propertylist_delete(plist);

        skip_if(0);

        cpl_frameset_insert(usedframes, cpl_frame_duplicate(ifrm));
        cpl_frameset_insert(usedframes, cpl_frame_duplicate(efrm));
        /* wait for child to send of all resampling jobs before starting the next
         * this is to assign as many cpus as possible to handling one input
         * file to reduce disk IO and load balance the coaddition phase */
        sem_wait(sem_coadd);
        pids[i++] = run_swarp_coadd(ifrm, efrm, parlist,
                                    sem_coadd, sem_resamp);
        cpl_msg_info(cpl_func, "Coadding file (%d/%d)", i, nfiles);
    }
    for (int j = 0; j < i; j++) {
        int status;
        pid_t r = waitpid(-1, &status, 0);
        error_if(r == -1 || !WIFEXITED(status) || WEXITSTATUS(status),
                 CPL_ERROR_FILE_IO, "Coaddition failed: %d, %s",
                 WEXITSTATUS(status), r == -1 ? strerror(errno) : "");
    }

    end_skip;
    sem_close(sem_coadd);
    sem_close(sem_resamp);
    cpl_free(sem_resamp_name);
    cpl_free(sem_coadd_name);

    return cpl_error_get_code();
}


static cpl_error_code
save_single_images(cpl_frameset * framelist, const cpl_size nfiles,
                   const pid_t * pids, const cpl_frameset * usedframes,
                   const cpl_parameterlist * parlist)
{
    cpl_image * img = NULL;
    cpl_image * wgt = NULL;
    cpl_propertylist * plist = NULL;

    for (cpl_size j = 0; j < nfiles; j++) {
        char coadd_img[200];
        char coadd_wgt[200];

        sprintf(coadd_img, "tmp_coadd_%d.fits", (int)pids[j]);
        sprintf(coadd_wgt, "tmp_coadd_wgt_%d.fits", (int)pids[j]);

        cpl_image_delete(img);
        cpl_image_delete(wgt);
        img = cpl_image_load(coadd_img, CPL_TYPE_UNSPECIFIED, 0, 0);
        wgt = cpl_image_load(coadd_wgt, CPL_TYPE_UNSPECIFIED, 0, 0);
        skip_if(0);

        cpl_propertylist_delete(plist);
        plist = cpl_propertylist_load(coadd_img, 0);
        skip_if(0);

        sprintf(coadd_img, "swarp_coadd_img_%03d.fits", (int)j);
        sprintf(coadd_wgt, "swarp_coadd_wgt_%03d.fits", (int)j);

        skip_if(irplib_dfs_save_image(framelist, parlist, usedframes,
                                      img, CPL_BPP_IEEE_FLOAT,
                                      RECIPE_STRING, "COADDED_IMAGE",
                                      plist, NULL, visir_pipe_id,
                                      coadd_img));
        skip_if(irplib_dfs_save_image(framelist, parlist, usedframes,
                                      wgt, CPL_BPP_IEEE_FLOAT,
                                      RECIPE_STRING, "COADDED_WEIGHT",
                                      plist, NULL, visir_pipe_id,
                                      coadd_wgt));
    }

    end_skip;
    cpl_image_delete(img);
    cpl_image_delete(wgt);
    cpl_propertylist_delete(plist);

    return cpl_error_get_code();
}


static cpl_propertylist *
generate_beam_averages(cpl_image ** av, cpl_image ** wav, float * imgwgts,
                       const cpl_size nfiles,
                       const pid_t * pids, const unsigned long * beamids)
{
    /* all files are in the same projection so simply add them together */
    cpl_image * img = NULL;
    cpl_image * wgt = NULL;
    cpl_propertylist * plist = NULL;
    unsigned long max_beamid = 0;
    float tmpwgts[nfiles];

    cpl_ensure(nfiles > 0, CPL_ERROR_ILLEGAL_INPUT, NULL);

    for (cpl_size j = 0; j < nfiles; j++) {
        const unsigned long idx = beamids[j];
        char coadd_img[200];
        char coadd_wgt[200];

        cpl_image_delete(img);
        cpl_image_delete(wgt);
        sprintf(coadd_img, "tmp_coadd_%d.fits", (int)pids[j]);
        sprintf(coadd_wgt, "tmp_coadd_wgt_%d.fits", (int)pids[j]);
        img = cpl_image_load(coadd_img, CPL_TYPE_UNSPECIFIED, 0, 0);
        wgt = cpl_image_load(coadd_wgt, CPL_TYPE_UNSPECIFIED, 0, 0);

        skip_if(0);

        if (plist == NULL)
            plist = cpl_propertylist_load(coadd_img, 0);

        /* PIXEL mode is significantly faster and accurate enough for the
         * fov of VISIR, set it back to TAN for other tools,
         * note that swarp will flip the image and CD1_1 will have a different
         * sign to account for that */
        if (strcmp(cpl_propertylist_get_string(plist, "CTYPE1"),
                   "PIXEL") == 0) {
            cpl_propertylist_set_string(plist, "CTYPE1", "RA---TAN");
            cpl_propertylist_set_string(plist, "CTYPE2", "DEC--TAN");
            cpl_propertylist_set_string(plist, "CUNIT1", "deg");
            cpl_propertylist_set_string(plist, "CUNIT2", "deg");
        }

        skip_if(0);

        cpl_image_multiply(img, wgt);
        if (av[idx] == NULL) {
            av[idx] = cpl_image_duplicate(img);
            wav[idx] = cpl_image_duplicate(wgt);
        }
        else {
            skip_if(cpl_image_add(av[idx], img));
            skip_if(cpl_image_add(wav[idx], wgt));
        }
        max_beamid = CX_MAX(max_beamid, idx);
        /* weights always the same for a single beam */
        tmpwgts[idx] = imgwgts[j];
    }

    for (unsigned long i = 0; i < max_beamid + 1; i++) {
        cpl_image_divide(av[i], wav[i]);
        imgwgts[i] = tmpwgts[i];
    }

    end_skip;

    cpl_image_delete(img);
    cpl_image_delete(wgt);

    return plist;
}


/*----------------------------------------------------------------------------*/
/**
  @internal
  @brief    generate average image from all beam averages
  @param    avg  beam averages
  @param    wgt  beam weights
  @param    imgwgts beam weights
  @param    n number of beams
  @param    oimg  output average
  @param    owgt  output weight
  @param    octr  output contribution map
  @param    ctype combination type
  @return   0 iff everything is ok
*/
/*----------------------------------------------------------------------------*/
static cpl_error_code
generate_combined_image(cpl_image ** avg, cpl_image ** wgt,
                        const float * imgwgts, const unsigned long n,
                        cpl_image ** oimg, cpl_image ** owgt,
                        cpl_image ** octr, visir_comb_type ctype)
{
    /* summing combine mode will lead to ugly images in case of bad pixels */
    cpl_image * cimg = cpl_image_new(cpl_image_get_size_x(avg[0]),
                                     cpl_image_get_size_y(avg[0]),
                                     CPL_TYPE_FLOAT);
    cpl_image * cwgt = cpl_image_new(cpl_image_get_size_x(wgt[0]),
                                     cpl_image_get_size_y(wgt[0]),
                                     CPL_TYPE_FLOAT);
    cpl_image * contrib = cpl_image_new(cpl_image_get_size_x(wgt[0]),
                                        cpl_image_get_size_y(wgt[0]),
                                        CPL_TYPE_INT);
    int wgtsum = 0;

    skip_if(0);
    /* e_sum = sqrt(sum(sig^2)) / n        :sig = sqrt(1/w)
       w_sum = 1 / ((sum(1/w) / n ^ 2)) */
    for (unsigned long i = 0; i < n; i++) {
        cpl_mask * m = cpl_mask_threshold_image_create(wgt[i], -1, FLT_EPSILON);
        cpl_image * bad = cpl_image_new_from_mask(m);
        cpl_image_reject_from_mask(avg[i], m);
        /*  the beams have a non zero background which averages out over
         * all beams, but in the case of bad pixels some values are missing
         * which leads to artefacts if we don't correct the background */
        cpl_image_subtract_scalar(avg[i], cpl_image_get_median(avg[i]));
        cpl_image_fill_rejected(avg[i], 0);
        cpl_mask_delete(m);

        /* will leave zero weight bad pixels at zero */
        cpl_image_power(wgt[i], -1);
        cpl_image_accept_all(wgt[i]);

        cpl_image_multiply_scalar(bad, imgwgts[i]);
        cpl_image_subtract(contrib, bad);
        cpl_image_delete(bad);

        skip_if(0);

        cpl_image_add(cimg, avg[i]);
        cpl_image_add(cwgt, wgt[i]);

        wgtsum += imgwgts[i];
    }

    cpl_image_add_scalar(contrib, wgtsum);
    if (ctype == VISIR_AVG_COMB) {
        cpl_image_divide(cimg, contrib);

        cpl_image_divide(cwgt, contrib);
        cpl_image_divide(cwgt, contrib);
    }
    cpl_image_power(cwgt, -1);

    *oimg = cimg;
    *owgt = cwgt;
    *octr = contrib;

    end_skip;

    return cpl_error_get_code();
}


/* ---------------------------------------------------------------------------*/
/**
 * @brief cut low contribution edges from image
 *
 * @param img pointer to data image, replaced by new image
 * @param wgt pointer to weight image, replaced by new image
 * @param ctr pointer to contribution image, replaced by new image
 *
 * remove edges with less than 70% contribution, this effectively removes empty
 * edges create by the resampling kernel and one additional row where the
 * contribution might be very low due to shift-and-add
 */
/* ---------------------------------------------------------------------------*/
static cpl_error_code
cut_bad_edges(cpl_image ** img, cpl_image  ** wgt, cpl_image ** ctr,
              cpl_propertylist * plist)
{
    const cpl_size nx = cpl_image_get_size_x(*ctr);
    const cpl_size ny = cpl_image_get_size_y(*ctr);
    const cpl_size max_contrib = cpl_image_get_max(*ctr);
    const double ctr_cutoff = max_contrib == 2 ? 0.4 : 0.7;
    cpl_size cutly = 0;
    cpl_size cutuy = 0;
    cpl_size cutlx = 0;
    cpl_size cutux = 0;
    for (cpl_size i = 0; i < ny / 2; i++) {
        cpl_vector * row = cpl_vector_new_from_image_row(*ctr, i + 1);
        double sum = cpl_vector_get_sum(row);
        cpl_vector_delete(row);
        if (sum >= max_contrib * nx * ctr_cutoff) {
            break;
        }
        cutly = i + 1;
    }
    for (cpl_size i = ny - 1; i >= ny / 2; i--) {
        cpl_vector * row = cpl_vector_new_from_image_row(*ctr, i + 1);
        double sum = cpl_vector_get_sum(row);
        cpl_vector_delete(row);
        if (sum >= max_contrib * nx * ctr_cutoff) {
            break;
        }
        cutuy = i + 1;
    }
    for (cpl_size i = 0; i < nx / 2; i++) {
        cpl_vector * col = cpl_vector_new_from_image_column(*ctr, i + 1);
        double sum = cpl_vector_get_sum(col);
        cpl_vector_delete(col);
        if (sum >= max_contrib * ny * ctr_cutoff) {
            break;
        }
        cutlx = i + 1;
    }
    for (cpl_size i = nx - 1; i >= nx / 2; i--) {
        cpl_vector * col = cpl_vector_new_from_image_column(*ctr, i + 1);
        double sum = cpl_vector_get_sum(col);
        cpl_vector_delete(col);
        if (sum >= max_contrib * ny * ctr_cutoff) {
            break;
        }
        cutux = i + 1;
    }
    cutlx += 1;
    cutly += 1;
    cutux -= 1;
    cutuy -= 1;
    cpl_msg_info(cpl_func, "Cutting low contribution edges "
                 "llx: %lld, lly: %lld, urx: %lld, ury: %lld",
                 cutlx, cutly, cutux, cutuy);

    /* make sure to only cut if something remains */
    cpl_image * nimg, * nwgt, * nctr;
    if (cutlx < cutux && cutly < cutuy) {
        cpl_propertylist_set_double(plist, "CRPIX1",
                        cpl_propertylist_get_double(plist, "CRPIX1") - cutlx);
        cpl_propertylist_set_double(plist, "CRPIX2",
                        cpl_propertylist_get_double(plist, "CRPIX2") - cutly);
        nimg = cpl_image_extract(*img, cutlx, cutly, cutux, cutuy);
        nwgt = cpl_image_extract(*wgt, cutlx, cutly, cutux, cutuy);
        nctr = cpl_image_extract(*ctr, cutlx, cutly, cutux, cutuy);
    }
    else {
        nimg = cpl_image_duplicate(*img);
        nwgt = cpl_image_duplicate(*wgt);
        nctr = cpl_image_duplicate(*ctr);
    }
    cpl_image_delete(*img);
    cpl_image_delete(*wgt);
    cpl_image_delete(*ctr);
    *img = nimg;
    *wgt = nwgt;
    *ctr = nctr;

    return cpl_error_get_code();
}


static cpl_error_code
check_inputs(const cpl_frameset * framelist, const char * config)
{
    irplib_framelist * alldata = irplib_framelist_cast(framelist);
    irplib_framelist * data = irplib_framelist_extract(alldata,
                                                       VISIR_UTIL_CORRECTED);
    irplib_framelist * errors = irplib_framelist_extract(alldata, "ERROR_MAP");
    irplib_framelist_load_propertylist_all(data, 0, "ESO QC", CPL_FALSE);
    error_if(irplib_framelist_get_size(data) !=
             irplib_framelist_get_size(errors), CPL_ERROR_ILLEGAL_INPUT,
             "Unequal number of data and error frames");
    skip_if(irplib_framelist_contains(data, "ESO QC BEAMID", CPL_TYPE_STRING,
                                      CPL_FALSE, 0.0));

    error_if(access(config, R_OK) != 0, CPL_ERROR_FILE_NOT_FOUND,
             "Swarp configuration not found: %s", config);
    end_skip;

    irplib_framelist_delete(alldata);
    irplib_framelist_delete(data);
    irplib_framelist_delete(errors);
    return cpl_error_get_code();
}


/*----------------------------------------------------------------------------*/
/**
  @brief    The recipe data reduction part is implemented here
  @param    framelist   the frames list
  @param    parlist     the parameters list
  @return   0 iff everything is ok
  */
/*----------------------------------------------------------------------------*/
static int visir_util_run_swarp(cpl_frameset            * framelist,
                                const cpl_parameterlist * parlist)
{
    cpl_frameset * usedframes = cpl_frameset_new();
    cpl_propertylist * plist = NULL;
    cx_list * imglist = cx_list_new();
    cx_list * errlist = cx_list_new();

    skip_if(check_swarp());
    /* only reading single extension files, no cache needed */
    cpl_fits_set_mode(CPL_FITS_STOP_CACHING);

    const cpl_boolean output_all =
        irplib_parameterlist_get_bool(parlist, PACKAGE,
                                      RECIPE_STRING, "output-all");
    const char * config =
        irplib_parameterlist_get_string(parlist, PACKAGE,
                                        RECIPE_STRING, "config_fname");
    const char * extra =
        irplib_parameterlist_get_string(parlist, PACKAGE,
                                        RECIPE_STRING, "extra_config");
    int nmax_procs =
        irplib_parameterlist_get_int(parlist, PACKAGE,
                                     RECIPE_STRING, "nprocs");
    if (visir_str_par_is_empty(extra))
        extra = "";

    if (nmax_procs <= 0) {
        nmax_procs = visir_get_num_threads(CPL_TRUE);
    }

    error_if(visir_str_par_is_empty(config), CPL_ERROR_FILE_NOT_FOUND,
             "A swarp configuration file is required. "
             "You can create a default configuration with: swarp -d");

    skip_if(check_inputs(framelist, config));

    unlink(SWARP_DATA_FILES);
    unlink(SWARP_COADD_HEAD);

    skip_if(0);

    create_swarp_frames(imglist, errlist, framelist);
    error_if(cx_list_size(imglist) == 0 || cx_list_size(errlist) == 0,
             CPL_ERROR_ILLEGAL_INPUT, "No frames tagged "
             VISIR_UTIL_CORRECTED" and "VISIR_UTIL_ERROR_MAP" found");

    skip_if(generate_coaddition_header(imglist, config, extra));

    {
        const cpl_size nfiles = cx_list_size(imglist);
        pid_t pids[nfiles];
        unsigned long beamids[nfiles];
        float imgwgts[nfiles];
        unsigned long nmax_beamid = 0;
        cpl_image * av[nfiles];
        cpl_image * wav[nfiles];
        cpl_image * cimg = NULL;
        cpl_image * cwgt = NULL;
        cpl_image * cctr = NULL;

        for (cpl_size j = 0; j < nfiles; j++)
            av[j] = wav[j] = NULL;

        skip_if(coadd_images(pids, beamids, imgwgts, usedframes,
                             imglist, errlist, nmax_procs, parlist));

        for (cpl_size i = 0; i < nfiles; i++)
            nmax_beamid = CX_MAX(nmax_beamid, beamids[i] + 1);

        error_if((cpl_size)nmax_beamid - 1 > nfiles,
                 CPL_ERROR_UNSUPPORTED_MODE,
                 "Only consecutive BEAMID's supported");

        if (output_all && nfiles > 1)
            save_single_images(framelist, nfiles, pids, usedframes, parlist);

        skip_if(0);

        plist = generate_beam_averages(av, wav, imgwgts, nfiles, pids, beamids);
        skip_if(plist == NULL);

        generate_combined_image(av, wav, imgwgts, nmax_beamid, &cimg, &cwgt,
                                &cctr, VISIR_AVG_COMB);
        for (cpl_size i = 0; i < nfiles; i++) {
            cpl_image_delete(av[i]);
            cpl_image_delete(wav[i]);
        }

        cut_bad_edges(&cimg, &cwgt, &cctr, plist);

        irplib_dfs_save_image(framelist, parlist, usedframes, cimg,
                              CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
                              VISIR_IMG_COADDED_IMG, plist, NULL,
                              visir_pipe_id, "swarp_coadd_img_all.fits");
        irplib_dfs_save_image(framelist, parlist, usedframes, cwgt,
                              CPL_BPP_IEEE_FLOAT, RECIPE_STRING,
                              VISIR_IMG_COADDED_WGT, plist, NULL,
                              visir_pipe_id, "swarp_coadd_wgt_all.fits");
        irplib_dfs_save_image(framelist, parlist, usedframes, cctr,
                              CPL_TYPE_INT, RECIPE_STRING,
                              VISIR_IMG_COADDED_CTR, plist, NULL,
                              visir_pipe_id, "swarp_coadd_ctr_all.fits");
        cpl_image_delete(cimg);
        cpl_image_delete(cwgt);
        cpl_image_delete(cctr);
        skip_if(0);
    }

    end_skip;

    unlink(SWARP_DATA_FILES);
    unlink(SWARP_COADD_HEAD);
    cx_list_destroy(imglist, (cx_free_func)cpl_frame_delete);
    cx_list_destroy(errlist, (cx_free_func)cpl_frame_delete);
    cpl_frameset_delete(usedframes);
    cpl_propertylist_delete(plist);
    return cpl_error_get_code();
}
