/**@file
 *
 * Given a matrix A, compute an ILU(k) preconditioner for A.
 *
 *
 *      authors: Claus-Justus Heine
 *               Abteilung fuer Angewandte Mathematik                     
 *               Albert-Ludwigs-Universitaet Freiburg
 *               Hermann-Herder-Str. 10
 *               79104 Freiburg
 *               Germany
 *               claus@mathematik.uni-freiburg.de
 *
 *      Copyright (c) by C.-J. Heine (2003-2009)
 *
 * @a ILU means Incomplete Lower-Upper factorization. @a (k) indicates
 * that the fill in (i.e. the non-zero positions in the factorization
 * which are not present in the original matrix) is controlled by a
 * (simple) combinatorical algorithm based on the fill structure
 * (profile) of the original matrix.
 *
 * The implementation presented here is an incomplete version of the
 * following standard "left-looking" factorization algorithm:
 *
 * for k=1,n
 *   for j=1,k-1
 *     for i=j+1,n
 *       a(k,i) = a(k,i) - a(k,j)*a(j,i)
 *     end for
 *   end for
 *   a(k,k) = 1/a(k,k)
 *   for j=k+1,n
 *     a(k,j) = a(k,k) * a(k,j)
 *   end for
 * end for
 *
 * We do NOT use ALBERTA format, but compressed row format. We need
 * additional information about the fill-levels anyway.
 */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

#include <alberta.h>

#include "crs_matrix.h"
#if HAVE_LIBGPSKCA
# include "optimize_profile.h"
#endif
#include "ilu_k_precon.h"

#include <sys/time.h>

#ifndef IF_INFO
# define IF_INFO(info, level) if (msg_info && info > level)
#endif
#ifndef INFO_VERBOSE
# define INFO_VERBOSE 5
#endif
#ifndef INFO_DEBUG
# define INFO_DEBUG 10
#endif

#define ILU_K_OPTIMIZE_PROFILE 1
#define ILU_K_REVERSE          0
#define DEBUG_ILU_K            0
#define CLEAR_DENSE_ROW        0

typedef struct sparse_row {
  int   size;
  REAL  *val;
  int   *col;
} SPARSE_ROW;
/**@endcond */

/* <<< helper functions (spare/dense row conversion etc.) */

/* <<< sparse_row_accumulate_profile() */

/* This function is called with A[dof][row] != 0, and we have to
 * evaluate the expression A[dof][row]*ilu[row][ind] != 0. So we get
 * fill-in when ilu[row][ind] != 0.
 *
 * If w_lev[ind] < 0 then just increment the level, i.e. set the level
 * to 0. This means: if an element in A is non-zero then it will be
 * non-zero in the ILU preconditioner.
 *
 * if w_lev[ind] >= 0 then some other row of ilu has already caused
 * fill-in. Then: if our level is smaller we win (because there should
 * be fill-in if any row's fill-level is small enough).  If our level
 * is bigger, just don't change w_lev[ind].
 */
static inline void sparse_row_accumulate_profile(const CRS_MATRIX *ilu,
						 int row,
						 int *w_lev)
{
  int ind, new_level;
  CRS_OFFSET i;
  int *entries = (int *)ilu->entries; /* abuse the entries vector, this only
				       * works when sizeof(int) <=
				       * ilu->entry_size
				       */

  for (i = CRS_U(ilu, row); i < CRS_END(ilu, row); i++) {
    ind = ilu->info->col[i];
    new_level = 1 + MAX(w_lev[row],entries[i]);
    if (w_lev[ind] < 0 || new_level < w_lev[ind]) {
      w_lev[ind] = new_level;
    }
  }
}

/* >>> */

/* <<< dense_to_space_row() */

/* Create a new row in ilu. Discard all entries with w_lev[i] > level.
 * Note: this appends a new row to ilu, so we don't need to shift
 * memory around.
 */
static inline void dense_to_sparse_row(CRS_MATRIX_INFO *info,
				       CRS_MATRIX *ilu,
				       int row, int PIrow,
				       int level,
				       int *w_lev,
				       int bw)
{
  int i, start, stop;

  crs_matrix_info_alloc_check(info);
  crs_matrix_alloc_check(ilu);

  info->n_entries ++;
  if (!info->bound || info->bound->vec[PIrow] < DIRICHLET) {
    start = MAX(row-bw, 0);
    for (i = start; i < row; i++) {
      if (w_lev[i] >= 0 && w_lev[i] <= level) {
	crs_matrix_info_alloc_check(info);
	crs_matrix_alloc_check(ilu);      

	info->col[info->n_entries] = i;
	CRS_VALUE(ilu, info->n_entries, int) = w_lev[i];
	info->n_entries ++;
      }
      w_lev[i] = -1;
    }
    w_lev[i] = -1; /* don't forget the diagonal entry ;) */
    CRS_U(ilu, row) = info->n_entries;
    CRS_VAL_REL(ilu, row, 0, int) = 0;
    stop = MIN(row+bw+1, info->dim);
    for (i = row + 1; i < stop; i++) {
      if (w_lev[i] >= 0 && w_lev[i] <= level) {
	crs_matrix_info_alloc_check(info);
	crs_matrix_alloc_check(ilu);

	info->col[info->n_entries] = i;
	CRS_VALUE(ilu, info->n_entries, int) = w_lev[i];
	info->n_entries ++;
      }
      w_lev[i] = -1;
    }
#if DEBUG_ILU_K
    for (i = 0; i < start; i++) {
      if (w_lev[i] != -1) {
	ERROR_EXIT("w_lev garbled at row %d, entry %d (bw: %d, dim %d)\n",
		   row, i, bw, info->dim);
      }
    }
    for (i = stop; i < info->dim; i++) {
      if (w_lev[i] != -1) {
	ERROR_EXIT("w_lev garbled at row %d, entry %d (bw: %d, dim %d)\n",
		   row, i, bw, info->dim);
      }
    }
#endif
  } else {
    CRS_U(ilu, row) = info->n_entries;
  }
  CRS_END(ilu, row) = info->n_entries; /* mark end of row */
}

/* >>> */

/* <<< sparse_to_dense_row() */

/* copy this row into the dense vector w. As we know about the bandwidth
 * of A, we could optimize this.
 */
static inline void sparse_to_dense_row(int start, int stop,
				       int *Acol,
				       CRS_MATRIX_INFO *info,
				       int dof, int *w_lev)
{
  int l;
  S_CHAR *bound = info->bound ? info->bound->vec : NULL;

  if (bound) {
    for (l = start; l < stop; l++) {
      if (bound[Acol[l]] < DIRICHLET) {
	w_lev[info->P[Acol[l]]] = 0;
      }
    }
  } else {
    for (l = start; l < stop; l++) {
      w_lev[info->P[Acol[l]]] = 0;
    }
  }
}

/* >>> */

/* <<< matrix_row_to_dense_row() */

/* Copy this row into the dense vector w. As we know about the
 * bandwidth of A, we could optimize this. This version is for
 * ALBERTA's fancy MATRIX_ROW structure.
 */
static inline void matrix_row_to_dense_row(MATRIX_ROW *Arow,
					   CRS_MATRIX_INFO *info,
					   int dof, int *w_lev)
{
  S_CHAR *bound = info->bound ? info->bound->vec : NULL;

  if (bound) {
    FOR_ALL_MAT_COLS(REAL, Arow,
		     if (bound[col_dof] < DIRICHLET) {
		       w_lev[info->P[col_dof]] = 0;
		     });
  } else {
    FOR_ALL_MAT_COLS(REAL, Arow, w_lev[info->P[col_dof]] = 0);
  }
}

/* >>> */

/* >>> */

/* <<< generate the profile of the preconditioner */

/* <<< ilu_k_create_profile(), CRS-matrix version */

/** This one just creates the sparse structure of the ILU(k)
 * matrix.
 *
 * @param[in] A      The CRS-matrix to factorize (incompletely).
 * @param[in] level  The maximal fill-level for the incomplete
 *                   factorization.
 * @param[in] info_level How much noise to make during the
 *                   factorization process.
 *
 * @return A CRS-matrix structure. Only the connectivity structure of
 *         the matrix is initialised at this point. Call
 *         ilu_k_create() and ilu_k_create_dd() to generate the actual
 *         preconditioner.
 */
CRS_MATRIX *ilu_k_create_profile(const CRS_MATRIX *A,
				 int level, int info_level)
{
  FUNCNAME("ilu_k_create_profile");
  CRS_MATRIX_INFO *info;
  CRS_MATRIX *ilu;
  static int *w_lev;
  static int  w_lev_size;
  int         l, dof, bandwidth;
  int64_t     profile;
  int         *P, *PI;
  S_CHAR      *bound = A->info->bound ? A->info->bound->vec : NULL;
  struct timeval real_tv;
  REAL           real_t = 0.0;
  
  IF_INFO(info_level, INFO_VERBOSE) {
    MSG("Start for %s\n", A->name);
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }
  INFO(info_level, INFO_DEBUG, "");

  P  = MEM_ALLOC(A->info->dim, int);
  PI = MEM_ALLOC(A->info->dim, int);
#if ILU_K_OPTIMIZE_PROFILE && HAVE_LIBGPSKCA
  crs_matrix_optimize_profile(A, P, PI, &bandwidth, &profile, true, info_level);
  if (profile < 0) {
    profile = A->info->n_entries*5;
  }
#elif ILU_K_REVERSE
  for (l = 0; l < A->info->dim; l++) {
    P[A->info->dim - l - 1] = l;
  }
  for (l = 0; l < A->info->dim; l++) {
    PI[P[l]] = l;
  }
  profile = A->info->n_entries*5;
  bandwidth = A->info->dim;
#else
  for (l = 0; l < A->info->dim; l++) {
    P[l] = l;
  }
  for (l = 0; l < A->info->dim; l++) {
    PI[P[l]] = l;
  }
  profile = A->info->n_entries*5;
  bandwidth = A->info->dim;
#endif
  info       =
    (CRS_MATRIX_INFO *)crs_matrix_info_alloc(A->info->dim,
					     A->info->dim + profile/5);

  info->bound = A->info->bound;
  info->P    = P;
  info->PI   = PI;
  ilu        = crs_matrix_get(A->name, A->entry_size, NULL, NULL, NULL, info);

  CRS_BEG(ilu, 0) = 0;

  if (w_lev_size < info->dim) {
    w_lev = MEM_REALLOC(w_lev, w_lev_size, info->dim, int);
    w_lev_size = info->dim;
  }

  for (l = 0; l < info->dim; l++) {
    w_lev[l] = -1;
  }

  for (dof = 0; dof < info->dim; dof ++) {
    int PIrow = PI[dof];
    int start = CRS_BEG(A, PIrow);
    int stop  = CRS_END(A, PIrow);  

    if (bound) {
      if (bound[PIrow] < DIRICHLET) {
	sparse_to_dense_row(start, stop, A->info->col, info, dof, w_lev);
	/* now the profile of A is stored in w. w[i] == 0 <-> A[dof][i] != 0,
	 * w[i] == -1 <-> A[dof][i] == 0
	 */

	/* loop again over the row and accumulate factorization and levels
	 * in w.
	 */
#if DEBUG_ILU_K
	for (l = 0; l < MAX(0, dof-bandwidth); l++) {
	  if (w_lev[l] != -1) {
	    ERROR_EXIT("w_lev[%d] != -1, dof = %d, bw = %d\n",
		       l, dof, bandwidth);
	  }
	}
#endif
	for (l = MAX(0, dof - bandwidth); l < dof; l++) {
	  if (w_lev[l] >= 0 && bound[PI[l]] < DIRICHLET) {
	    sparse_row_accumulate_profile(ilu, l, w_lev);
	  }
	}
      }
    } else {
      sparse_to_dense_row(start, stop, A->info->col, info, dof, w_lev);
      /* now the profile of A is stored in w. w[i] == 0 <-> A[dof][i] != 0,
       * w[i] == -1 <-> A[dof][i] == 0
       */

      for (l = MAX(0, dof - bandwidth); l < dof; l++) {
	if (w_lev[l] >= 0) {
	  sparse_row_accumulate_profile(ilu, l, w_lev);
	}
      }
    }

    /* insert w into ilu */
    dense_to_sparse_row(info, ilu, dof, PIrow, level, w_lev, bandwidth);
    IF_INFO(info_level, INFO_DEBUG) {
      if (dof % 100 == 0) {
	printf(".");
      }
    }
  }

#if DEBUG_ILU_K
  for (l = 0; l < info->dim; l++) {
    if (w_lev[l] != -1) {
      ERROR_EXIT("Ooops, ilu(k) garbled\n");
    }
  }
#endif

  crs_matrix_info_trim(info);

  /* Permute the columns, this sparse some indirect adress
   * computations. The diagonal index must not be touched.
   */
  for (dof = 0; dof < ilu->info->dim; dof++) {
    int ioff;
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      ilu->info->col[ioff] = PI[ilu->info->col[ioff]];
    }
  }

  INFO(info_level, INFO_DEBUG, "\n");
  IF_INFO(info_level, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Stop for %s ILU(%d)\n", A->name, level);
    MSG("Non-zero entries of system matrix: %d(%d)\n",
	ilu->info->n_entries, A->info->n_entries);
    MSG("Real time elapsed: %e\n", real_t);
  }

  return ilu;
}

/* >>> */

/* <<< ilu_k_dm_create_profile(), ALBERTA DOF_MATRIX version */

/** This one just creates the sparse structure of the ILU(k)
 * matrix.
 *
 * @param[in] A      The DOF_MATRIX to factorize (incompletely).
 * @param[in] level  The maximal fill-level for the incomplete
 *                   factorization.
 * @param[in] info_level How much noise to make during the
 *                   factorization process.
 *
 * @return A CRS-matrix structure. Only the connectivity structure of
 *         the matrix is initialised at this point. Call
 *         ilu_k_create() and ilu_k_create_dd() to generate the actual
 *         preconditioner.
 */
CRS_MATRIX *ilu_k_dm_create_profile(const DOF_MATRIX *A,
				    const DOF_SCHAR_VEC *user_mask,
				    int level, int info_level)
{
  FUNCNAME("ilu_k_create_profile_dm");
  CRS_MATRIX_INFO *info;
  CRS_MATRIX      *ilu;
  static int      *w_lev;
  static int      w_lev_size;
  int             l, dof, bandwidth, row_dim, hole_count;
  int64_t         profile;
  int             *P, *PI;
  S_CHAR          *bound;
  DOF_SCHAR_VEC   *mask;
  struct timeval  real_tv;
  REAL            real_t = 0.0;
  
  IF_INFO(info_level, INFO_VERBOSE) {
    MSG("Start for %s\n", A->name);
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }
  INFO(info_level, INFO_DEBUG, "");

  row_dim = A->row_fe_space->admin->size_used;

  P  = MEM_ALLOC(row_dim, int);
  PI = MEM_ALLOC(row_dim, int);

  mask = get_dof_schar_vec("temp mask", A->row_fe_space);

  hole_count = 0;
  if (user_mask != NULL) {
    for (l = 0; l < row_dim; l++) {
      if (A->matrix_row[l] == NULL) {
	mask->vec[l] = DIRICHLET;
	++hole_count;
      } else {
	mask->vec[l] = user_mask->vec[l];
      }
    }
  } else if (!BNDRY_FLAGS_IS_INTERIOR(A->dirichlet_bndry)) {
    dirichlet_bound(NULL, NULL, mask, A->dirichlet_bndry, NULL);
    for (l = 0; l < row_dim; l++) {
      if (A->matrix_row[l] == NULL) {
	++hole_count;
	mask->vec[l] = DIRICHLET;
      }
    }
  } else {
    for (l = 0; l < row_dim; l++) {
      if (A->matrix_row[l] == NULL) {
	++hole_count;
	mask->vec[l] = DIRICHLET;
      } else {
	mask->vec[l] = INTERIOR;
      }
    }
  }

  if (hole_count == 0 &&
      user_mask == NULL && BNDRY_FLAGS_IS_INTERIOR(A->dirichlet_bndry)) {
    free_dof_schar_vec(mask);
    mask = NULL;
  }

#if ILU_K_OPTIMIZE_PROFILE && HAVE_LIBGPSKCA
  dof_matrix_optimize_profile(A, mask, P, PI, &bandwidth, &profile, true,
			      info_level);
  if (profile < 0) {
    profile = A->n_entries*5;
  }
#elif ILU_K_REVERSE
  for (l = 0; l < row_dim; l++) {
    P[row_dim - l - 1] = l;
  }
  for (l = 0; l < row_dim; l++) {
    PI[P[l]] = l;
  }
  profile = A->n_entries*5;
  bandwidth = row_dim;
#else
  for (l = 0; l < row_dim; l++) {
    P[l] = l;
  }
  for (l = 0; l < row_dim; l++) {
    PI[P[l]] = l;
  }
  profile = A->n_entries*5;
  bandwidth = row_dim;
#endif
  info = (CRS_MATRIX_INFO *)crs_matrix_info_alloc(row_dim, row_dim + profile/5);
  info->bound = mask;

  bound = info->bound != NULL ? info->bound->vec : NULL;

  info->P     = P;
  info->PI    = PI;
  ilu = crs_matrix_get(A->name, MATENT_SIZE(A->type), NULL, NULL, NULL, info);

  CRS_BEG(ilu, 0) = 0;

  if (w_lev_size < info->dim) {
    w_lev = MEM_REALLOC(w_lev, w_lev_size, info->dim, int);
    w_lev_size = info->dim;
  }

  for (l = 0; l < info->dim; l++) {
    w_lev[l] = -1;
  }

  for (dof = 0; dof < info->dim; dof ++) {
    int PIrow = PI[dof];

    if (bound) {
      if (bound[PIrow] < DIRICHLET) {
	matrix_row_to_dense_row(A->matrix_row[PIrow], info, dof, w_lev);
	/* now the profile of A is stored in w. w[i] == 0 <-> A[dof][i] != 0,
	 * w[i] == -1 <-> A[dof][i] == 0
	 */

	/* loop again over the row and accumulate factorization and levels
	 * in w.
	 */
#if DEBUG_ILU_K
	for (l = 0; l < MAX(0, dof-bandwidth); l++) {
	  if (w_lev[l] != -1) {
	    ERROR_EXIT("w_lev[%d] != -1, dof = %d, bw = %d\n",
		       l, dof, bandwidth);
	  }
	}
#endif
	for (l = MAX(0, dof - bandwidth); l < dof; l++) {
	  if (w_lev[l] >= 0 && bound[PI[l]] < DIRICHLET) {
	    sparse_row_accumulate_profile(ilu, l, w_lev);
	  }
	}
      }
    } else {
      matrix_row_to_dense_row(A->matrix_row[PIrow], info, dof, w_lev);
      /* now the profile of A is stored in w. w[i] == 0 <-> A[dof][i] != 0,
       * w[i] == -1 <-> A[dof][i] == 0
       */

      for (l = MAX(0, dof - bandwidth); l < dof; l++) {
	if (w_lev[l] >= 0) {
	  sparse_row_accumulate_profile(ilu, l, w_lev);
	}
      }
    }

    /* insert w into ilu */
    dense_to_sparse_row(info, ilu, dof, PIrow, level, w_lev, bandwidth);
    IF_INFO(info_level, INFO_DEBUG) {
      if (dof % 100 == 0) {
	printf(".");
      }
    }
  }

#if DEBUG_ILU_K
  for (l = 0; l < info->dim; l++) {
    if (w_lev[l] != -1) {
      ERROR_EXIT("Ooops, ilu(k) garbled\n");
    }
  }
#endif

  crs_matrix_info_trim(info);

  /* Permute the columns, this sparse some indirect adress
   * computations. The diagonal index must not be touched.
   */
  for (dof = 0; dof < ilu->info->dim; dof++) {
    int ioff;
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      ilu->info->col[ioff] = PI[ilu->info->col[ioff]];
    }
  }

  INFO(info_level, INFO_DEBUG, "\n");
  IF_INFO(info_level, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Stop for %s ILU(%d)\n", A->name, level);
    MSG("Non-zero entries of system matrix: %d(%d)\n",
	ilu->info->n_entries, A->n_entries);
    MSG("Real time elapsed: %e\n", real_t);
  }

  if (mask != NULL) {
    free_dof_schar_vec(mask);
    info->bound = NULL;
  }

  return ilu;
}

/* >>> */

/* >>> */

/* <<< create the preconditioner matrix */

/* <<< CRS matrix versions */

/* <<< ilu_k_create_dd(), CRS- and DOF-matrix version */

/** Take the sparse matrix structure generated by
 * create_ilu_symbolic() and fill in the real values. Block matrix
 * version. This means we compute over the @f$DOW\times DOW@f$ matrix
 * ring.
 *
 * NOTE: we compute over @f$R^{DOW\times DOW}@f$.
 *
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     alpha Offset to add to the diagonal elements to be able
 *                      to factorize indefinite matrices.
 *
 * @param[in]     beta  Weight for off-diagonal elements when added to
 *                      diagonal elements. Intentionally used to
 *                      generate a preconditioner for indefinite
 *                      matrices.
 * @param[in]     info  Noisiness.
 *
 * @return -1 on error (singular diagonal), otherwise positive.
 */
int ilu_k_create_dd(const CRS_MATRIX *A,
		    CRS_MATRIX *ilu,
		    REAL alpha, REAL beta, int info)
{
  FUNCNAME("ilu_k_create_dd");
  int i, j, k, dof, PIrow;
  CRS_OFFSET ioff, l;
  int result = 0;
  static REAL_DD *w;
  static int     *w_flag;
  static int      w_size;
  struct timeval  real_tv;
  REAL            real_t = 0.0;

  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }

  if (w_size < ilu->info->dim) {
    w = MEM_REALLOC(w, w_size, ilu->info->dim, REAL_DD);
    w_flag = MEM_REALLOC(w_flag, w_size, ilu->info->dim, int);
    w_size = ilu->info->dim;
  }

  for (i = 0; i < ilu->info->dim; i++) {
    w_flag[i] = 0;
  }

  for (dof = 0; dof < ilu->info->dim; dof++) {

    /* copy this row into the dense vector w, and initialize remaining
     * values of w.
     */
    PIrow = ilu->info->PI[dof];
    for (l = CRS_BEG(A, PIrow); l < CRS_END(A, PIrow); l++) {
      int col = A->info->col[l];
      MCOPY_DOW((const REAL_D *)CRS_VALUE(A, l, REAL_DD), w[col]);
      w_flag[col] = 1;
    }
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      l = ilu->info->col[ioff];
      if (!w_flag[l]) {
	MSET_DOW(0.0, w[l]);
	w_flag[l] = 1;
      }
    }
    if (alpha != 0.0) {
      for (i = 0; i < DIM_OF_WORLD; i++) {
	w[PIrow][i][i] += alpha;
      }
    }
    /* accumulate factorization into w, discarding all fill-ins where
     * w_flag is not set.
     */
    for (j = CRS_BEG(ilu, dof) + 1; j < CRS_U(ilu, dof); j++) {
      k = ilu->info->col[j];

      for (ioff = CRS_U(ilu, k); ioff < CRS_END(ilu, k); ioff++) {
	if (w_flag[ilu->info->col[ioff]]) { /* entry used for fill-in */
	  REAL_DD tmp;

	  MM_DOW((const REAL_D *)w[k],
		 (const REAL_D *)CRS_VALUE(ilu, ioff, REAL_DD),
		 tmp);
	  if (ilu->info->col[ioff] == PIrow) {
	    MAXPY_DOW(-beta, (const REAL_D *)tmp, w[ilu->info->col[ioff]]);
	  } else {
	    MAXPY_DOW(-1.0, (const REAL_D *)tmp, w[ilu->info->col[ioff]]);
	  }
	}
      }
    }

    if (MDET_DOW((const REAL_D *)w[PIrow]) < 0.0) {
      MSG("Matrix \"%s\" not spd, row %d: "MFORMAT_DOW"\n",
	  ilu->name,
	  PIrow,
	  MEXPAND_DOW(w[PIrow]));
      result = -1;
      goto out;
    }
    /* copy result over to ilu->row[dof] and invert the pivot */
    MINVERT_DOW((const REAL_D *)w[PIrow], CRS_DIAG(ilu, dof, REAL_DD));
    w_flag[PIrow] = 0;
    for (i = CRS_BEG(ilu, dof) + 1; i < CRS_U(ilu, dof); i++) {
      MCOPY_DOW((const REAL_D *)w[ilu->info->col[i]],
		CRS_VALUE(ilu, i, REAL_DD));
      w_flag[ilu->info->col[i]] = 0;
    }
    /* normalize the row of U. sr->col[0] holds the index of the first
     * upper triangular element in this row. If there is no such
     * element, then sr->col[0] == sr->size.
     */
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      MM_DOW((const REAL_D *)CRS_DIAG(ilu, dof, REAL_DD),
	     (const REAL_D *)w[ilu->info->col[ioff]],
	     CRS_VALUE(ilu, ioff, REAL_DD));
      w_flag[ilu->info->col[ioff]] = 0;
    }
  }
  
 out:
  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Real time elapsed: %e\n", real_t);
  }
  return result;
}

/* >>> */

/* <<< ilu_k_create_adaptive_dd(), CRS-matrix version */

/** Adaptively try to generate an ILU(k) preconditioner. Try to
 * factorize @f$A + \alpha I@f$. Additionally, off-diagonal elements
 * are weighted by a factor @f$\beta < 1@f$.
 *
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     info  Noisiness.
 *
 * @return The number of iterations needed to adapt the off-diagonal
 *         weights and the diagonal offset.
 */
int ilu_k_create_dd_adaptive(const CRS_MATRIX *A, CRS_MATRIX *ilu, int info)
{
  FUNCNAME("ilu_k_create_dd_adaptive");
  REAL alpha = 0.0, beta = 1.0, coef = 1.0;
  int badness = 0;

  while (ilu_k_create_dd(A, ilu, alpha, beta, info) < 0) {
    coef  *= 2.0;
    alpha  = coef;
    /* beta   = 1.0/coef; */
    badness ++;
    INFO(info, INFO_VERBOSE,
	 "Using alpha = %e, beta = %e\n", alpha, beta);
  }
  return badness;
}

/* >>> */

/* <<< ilu_k_create(), CRS-matrix version */

/** Take the sparse matrix structure generated by
 * create_ilu_symbolic() and fill in the real values.
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     alpha Offset to add to the diagonal elements to be able
 *                      to factorize indefinite matrices.
 *
 * @param[in]     beta  Weight for off-diagonal elements when added to
 *                      diagonal elements. Intentionally used to
 *                      generate a preconditioner for indefinite
 *                      matrices.
 * @param[in]     info  Noisiness.
 *
 * @return -1 on error (singular diagonal), otherwise positive.
 */
int ilu_k_create(const CRS_MATRIX *A, CRS_MATRIX *ilu,
		 REAL alpha, REAL beta, int info)
{
  FUNCNAME("ilu_k_create");
  int i, j, k, l, dof, PIrow;
  CRS_OFFSET ioff, loff;
  int result = 0;
  static REAL *w;
  static int  *w_flag;
  static int   w_size;
  struct timeval real_tv;
  REAL           real_t = 0.0;
  REAL           diag_min = 1e304;

  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }

  if (w_size < ilu->info->dim) {
    w = MEM_REALLOC(w, w_size, ilu->info->dim, REAL);
    w_flag = MEM_REALLOC(w_flag, w_size, ilu->info->dim, int);
    w_size = ilu->info->dim;
  }

  for (i = 0; i < ilu->info->dim; i++) {
    w_flag[i] = 0;
  }

  for (dof = 0; dof < ilu->info->dim; dof++) {

    /* copy this row into the dense vector w, and initialize remaining
     * values of w.
     */
    PIrow = ilu->info->PI[dof];
    for (loff = CRS_BEG(A, PIrow); loff < CRS_END(A, PIrow); loff++) {
      int col     = A->info->col[loff];
      w[col]      = CRS_VALUE(A, loff, REAL);
      w_flag[col] = 1;
    }
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      l = ilu->info->col[ioff];
      if (!w_flag[l]) {
	w[l] = 0.0;
	w_flag[l] = 1;
      }
    }
    w[PIrow] += alpha;
    /* accumulate factorization into w, discarding all fill-ins where
     * w_flag is not set.
     */
    for (j = CRS_BEG(ilu, dof) + 1; j < CRS_U(ilu, dof); j++) {
      k = ilu->info->col[j];
      /* the "templates" algorithm has some little bugs here :-) */
      for (ioff = CRS_U(ilu, k); ioff < CRS_END(ilu, k); ioff++) {
	if (w_flag[ilu->info->col[ioff]]) { /* entry used for fill-in */
	  if (ilu->info->col[ioff] == PIrow) {
	    w[ilu->info->col[ioff]] -= beta * w[k] * CRS_VALUE(ilu, ioff, REAL);
	  } else {
	    w[ilu->info->col[ioff]] -= w[k] * CRS_VALUE(ilu, ioff, REAL);
	  }
	}
      }
    }

    if (w[dof] <= 0.0) {
      result = -1;
      MSG("Matrix \"%s\" not spd, row: %d, value: %e\n",
	  ilu->name, dof, w[dof]);
      goto out;
    }
    if (diag_min > w[PIrow]) {
      diag_min = w[PIrow];
    }
    /* copy result over to ilu->row[dof] and invert the pivot */
    CRS_DIAG(ilu, dof, REAL) = 1.0/w[PIrow];
    w_flag[PIrow] = 0;
#if CLEAR_DENSE_ROW
    w[PIrow] = 0.0;
#endif
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      CRS_VALUE(ilu, ioff, REAL) = w[ilu->info->col[ioff]];
#if CLEAR_DENSE_ROW
      w[ilu->info->col[ioff]] = 0.0;
#endif
      w_flag[ilu->info->col[ioff]] = 0;
    }
    /* normalize the row of U. sr->col[0] holds the index of the first
     * upper triangular element in this row. If there is no such
     * element, then sr->col[0] == sr->size.
     */
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      CRS_VALUE(ilu, ioff, REAL) *= CRS_DIAG(ilu, dof, REAL);
    }
  }
  
 out:
  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Real time elapsed: %e\n", real_t);
  }
  
  IF_INFO(info, INFO_DEBUG) {
    REAL min, max, absmin;
    REAL *entries = (REAL *)ilu->entries;

    min = max = entries[0];
    absmin = fabs(entries[0]);
    for (ioff = 1; ioff < ilu->info->n_entries; ioff++) {
      REAL val = entries[ioff];
      if (!isfinite(val)) {
	MSG("entries[%d] is not finite!\n", i);
      } else {
	if (val != 0.0 && val  > max) {
	  max = val;
	}
	if (val != 0.0 && val  < min) {
	  min = val;
	}
	if (val != 0.0 && fabs(val) < absmin) {
	  absmin = fabs(val);
	}
      }      
    }
    MSG("%s: min: %e, max: %e, absmin: %e\n", ilu->name, min, max, absmin);
    MSG("%s: diag min: %e\n", ilu->name, diag_min);
  }

  return result;
}

/* >>> */

/* <<< ilu_k_create_adaptive(), CRS_MATRIX version */

/** Adaptively try to generate an ILU(k) preconditioner. Try to
 * factorize @f$A + \alpha I@f$. Additionally, off-diagonal elements
 * are weighted by a factor @f$\beta < 1@f$.
 *
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     info  Noisiness.
 *
 * @return The number of iterations needed to adapt the off-diagonal
 *         weights and the diagonal offset.
 */
int ilu_k_create_adaptive(const CRS_MATRIX *A, CRS_MATRIX *ilu, int info)
{
  FUNCNAME("ilu_k_create_adaptive");
  REAL alpha = 0.0, beta = 1.0, coef = 1.0;
  int badness = 0;

  while (ilu_k_create(A, ilu, alpha, beta, info) < 0) {
    coef  *= 2.0;
    alpha  = coef;
    /* beta   = 1.0/coef; */
    badness ++;
    INFO(info, INFO_VERBOSE,
	 "Using alpha = %e, beta = %e\n", alpha, beta);
  }
  return badness;
}

/* >>> */

/* >>> */

/* <<< DOF matrix versions */

/* <<< ilu_k_dm_create_dd(), DOF-matrix version */

/** Take the sparse matrix structure generated by
 * create_ilu_symbolic() and fill in the real values. Block matrix
 * version. This means we compute over the @f$DOW\times DOW@f$ matrix
 * ring.
 *
 * NOTE: we compute over @f$R^{DOW\times DOW}@f$.
 *
 * @param[in]     A     The DOF-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     alpha Offset to add to the diagonal elements to be able
 *                      to factorize indefinite matrices.
 *
 * @param[in]     beta  Weight for off-diagonal elements when added to
 *                      diagonal elements. Intentionally used to
 *                      generate a preconditioner for indefinite
 *                      matrices.
 * @param[in]     info  Noisiness.
 *
 * @return -1 on error (singular diagonal), otherwise positive.
 */
int ilu_k_dm_create_dd(const DOF_MATRIX *A,
		       CRS_MATRIX *ilu,
		       REAL alpha, REAL beta, int info)
{
  FUNCNAME("ilu_k_dm_create_dd");
  int i, j, dof;
  CRS_OFFSET ioff, l;
  int result = 0;
  int *P = ilu->info->P, *PI = ilu->info->PI;
  static REAL_DD *w;
  static int     *w_flag;
  static int      w_size;
  struct timeval  real_tv;
  REAL            real_t = 0.0;

  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }

  if (w_size < ilu->info->dim) {
    w = MEM_REALLOC(w, w_size, ilu->info->dim, REAL_DD);
    w_flag = MEM_REALLOC(w_flag, w_size, ilu->info->dim, int);
    w_size = ilu->info->dim;
  }

  for (i = 0; i < ilu->info->dim; i++) {
    w_flag[i] = 0;
  }

  for (dof = 0; dof < ilu->info->dim; dof++) {
    /* copy this row into the dense vector w, and initialize remaining
     * values of w.
     */
    int PIrow = PI[dof];

    if (A->matrix_row[PIrow] == NULL) {
      SET_B_ID(ilu->entries, CRS_BEG(ilu, dof));
      continue;
    }

    FOR_ALL_MAT_COLS(REAL_DD, A->matrix_row[PIrow], {
	MCOPY_DOW((const REAL_D *)row->entry[col_idx], w[col_dof]);
	w_flag[col_dof] = 1;
      });
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      l = ilu->info->col[ioff];
      if (!w_flag[l]) {
	MSET_DOW(0.0, w[l]);
	w_flag[l] = 1;
      }
    }
    if (alpha != 0.0) {
      for (i = 0; i < DIM_OF_WORLD; i++) {
	w[PIrow][i][i] += alpha;
      }
    }
    /* accumulate factorization into w, discarding all fill-ins where
     * w_flag is not set.
     */
    for (j = CRS_BEG(ilu, dof) + 1; j < CRS_U(ilu, dof); j++) {
      int k  = ilu->info->col[j];
      int Pk = P[k];

      for (ioff = CRS_U(ilu, Pk); ioff < CRS_END(ilu, Pk); ioff++) {
	if (w_flag[ilu->info->col[ioff]]) { /* entry used for fill-in */
	  REAL_DD tmp;

	  MM_DOW((const REAL_D *)w[k],
		 (const REAL_D *)CRS_VALUE(ilu, ioff, REAL_DD),
		 tmp);
	  if (ilu->info->col[ioff] == PIrow) {
	    MAXPY_DOW(-beta, (const REAL_D *)tmp, w[ilu->info->col[ioff]]);
	  } else {
	    MAXPY_DOW(-1.0, (const REAL_D *)tmp, w[ilu->info->col[ioff]]);
	  }
	}
      }
    }

    if (MDET_DOW((const REAL_D *)w[PIrow]) <= 0.0) {
      MSG("Matrix \"%s\" not spd, row %d: "MFORMAT_DOW"\n",
	  ilu->name,
	  dof,
	  MEXPAND_DOW(w[PIrow]));
      result = -1;
      goto out;
    }
    /* copy result over to ilu->row[dof] and invert the pivot */
    MINVERT_DOW((const REAL_D *)w[PIrow], CRS_DIAG(ilu, dof, REAL_DD));
    w_flag[PIrow] = 0;
    for (i = CRS_BEG(ilu, dof) + 1; i < CRS_U(ilu, dof); i++) {
      MCOPY_DOW((const REAL_D *)w[ilu->info->col[i]],
		CRS_VALUE(ilu, i, REAL_DD));
      w_flag[ilu->info->col[i]] = 0;
    }
    /* normalize the row of U. sr->col[0] holds the index of the first
     * upper triangular element in this row. If there is no such
     * element, then sr->col[0] == sr->size.
     */
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      MM_DOW((const REAL_D *)CRS_DIAG(ilu, dof, REAL_DD),
	     (const REAL_D *)w[ilu->info->col[ioff]],
	     CRS_VALUE(ilu, ioff, REAL_DD));
      w_flag[ilu->info->col[ioff]] = 0;
    }
  }
  
 out:
  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Real time elapsed: %e\n", real_t);
  }
  return result;
}

/* >>> */

/* <<< ilu_k_dm_create_adaptive_dd(), DOF-matrix version */

/** Adaptively try to generate an ILU(k) preconditioner. Try to
 * factorize @f$A + \alpha I@f$. Additionally, off-diagonal elements
 * are weighted by a factor @f$\beta < 1@f$.
 *
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     info  Noisiness.
 *
 * @return The number of iterations needed to adapt the off-diagonal
 *         weights and the diagonal offset.
 */
int ilu_k_dm_create_dd_adaptive(const DOF_MATRIX *A, CRS_MATRIX *ilu, int info)
{
  FUNCNAME("ilu_k_dm_create_dd_adaptive");
  REAL alpha = 0.0, beta = 1.0, coef = 1.0;
  int badness = 0;

  while (ilu_k_dm_create_dd(A, ilu, alpha, beta, info) < 0) {
    coef  *= 2.0;
    alpha  = coef;
    /* beta   = 1.0/coef; */
    badness ++;
    INFO(info, INFO_VERBOSE,
	 "Using alpha = %e, beta = %e\n", alpha, beta);
  }
  return badness;
}

/* >>> */

/* <<< ilu_k_dm_create(), DOF-matrix version */

/** Take the sparse matrix structure generated by
 * create_ilu_symbolic() and fill in the real values.
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     alpha Offset to add to the diagonal elements to be able
 *                      to factorize indefinite matrices.
 *
 * @param[in]     beta  Weight for off-diagonal elements when added to
 *                      diagonal elements. Intentionally used to
 *                      generate a preconditioner for indefinite
 *                      matrices.
 * @param[in]     info  Noisiness.
 *
 * @return -1 on error (singular diagonal), otherwise positive.
 */
int ilu_k_dm_create(const DOF_MATRIX *A, CRS_MATRIX *ilu,
		    REAL alpha, REAL beta, int info)
{
  FUNCNAME("ilu_k_create");
  int i, j, l, dof;
  CRS_OFFSET ioff;
  int result = 0;
  int *P = ilu->info->P, *PI = ilu->info->PI;
  static REAL *w;
  static int  *w_flag;
  static int   w_size;
  struct timeval real_tv;
  REAL           real_t = 0.0;
  REAL           diag_min = 1e304;

  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t = -(REAL)real_tv.tv_sec - (REAL)real_tv.tv_usec*1e-6;
  }

  if (w_size < ilu->info->dim) {
    w = MEM_REALLOC(w, w_size, ilu->info->dim, REAL);
    w_flag = MEM_REALLOC(w_flag, w_size, ilu->info->dim, int);
    w_size = ilu->info->dim;
  }

  for (i = 0; i < ilu->info->dim; i++) {
    w_flag[i] = 0;
  }

  for (dof = 0; dof < ilu->info->dim; dof++) {
    int PIrow = PI[dof];

    if (A->matrix_row[PIrow] == NULL) {
      SET_S_ID(ilu->entries, CRS_BEG(ilu, dof));
      continue;
    }

    /* copy this row into the dense vector w, and initialize remaining
     * values of w.
     */
    FOR_ALL_MAT_COLS(REAL, A->matrix_row[PIrow], {
	w[col_dof]      = row->entry[col_idx];
	w_flag[col_dof] = 1;
      });
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      l = ilu->info->col[ioff];
      if (!w_flag[l]) {
	w[l] = 0.0;
	w_flag[l] = 1;
      }
    }
    w[PIrow] += alpha;
    /* accumulate factorization into w, discarding all fill-ins where
     * w_flag is not set.
     */
    for (j = CRS_BEG(ilu, dof) + 1; j < CRS_U(ilu, dof); j++) {
      int k  = ilu->info->col[j];
      int Pk = P[k]; /* only the columns of ILU are permuted ... */
      /* the "templates" algorithm has some little bugs here :-) */
      for (ioff = CRS_U(ilu, Pk); ioff < CRS_END(ilu, Pk); ioff++) {
	if (w_flag[ilu->info->col[ioff]]) { /* entry used for fill-in */
	  if (ilu->info->col[ioff] == PIrow) {
	    w[ilu->info->col[ioff]] -= beta * w[k] * CRS_VALUE(ilu, ioff, REAL);
	  } else {
	    w[ilu->info->col[ioff]] -= w[k] * CRS_VALUE(ilu, ioff, REAL);
	  }
	}
      }
    }

    if (w[PIrow] < 0.0) {
      result = -1;
      MSG("Matrix \"%s\" not spd, row: %d, value: %e\n",
	  ilu->name, PIrow, w[PIrow]);
      goto out;
    }
    if (diag_min > w[PIrow]) {
      diag_min = w[PIrow];
    }
    /* copy result over to ilu->row[dof] and invert the pivot */
    CRS_DIAG(ilu, dof, REAL) = 1.0/w[PIrow];
    w_flag[PIrow] = 0;
#if CLEAR_DENSE_ROW
    w[PIrow] = 0.0;
#endif
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_END(ilu, dof); ioff++) {
      CRS_VALUE(ilu, ioff, REAL) = w[ilu->info->col[ioff]];
#if CLEAR_DENSE_ROW
      w[ilu->info->col[ioff]] = 0.0;
#endif
      w_flag[ilu->info->col[ioff]] = 0;
    }
    /* normalize the row of U. sr->col[0] holds the index of the first
     * upper triangular element in this row. If there is no such
     * element, then sr->col[0] == sr->size.
     */
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      CRS_VALUE(ilu, ioff, REAL) *= CRS_DIAG(ilu, dof, REAL);
    }
  }
  
 out:
  IF_INFO(info, INFO_VERBOSE) {
    gettimeofday(&real_tv, NULL);
    real_t += (REAL)real_tv.tv_sec + (REAL)real_tv.tv_usec*1e-6;

    MSG("Real time elapsed: %e\n", real_t);
  }

  IF_INFO(info, INFO_DEBUG) {
    REAL min, max, absmin;
    REAL *entries = (REAL *)ilu->entries;

    min = max = entries[0];
    absmin = fabs(entries[0]);
    for (ioff = 1; ioff < ilu->info->n_entries; ioff++) {
      REAL val = entries[ioff];
      if (!isfinite(val)) {
	MSG("entries[%d] is not finite!\n", i);
      } else {
	if (val != 0.0 && val  > max) {
	  max = val;
	}
	if (val != 0.0 && val  < min) {
	  min = val;
	}
	if (val != 0.0 && fabs(val) < absmin) {
	  absmin = fabs(val);
	}
      }      
    }
    MSG("%s: min: %e, max: %e, absmin: %e\n", ilu->name, min, max, absmin);
    MSG("%s: diag min: %e\n", ilu->name, diag_min);
  }

  return result;
}

/* >>> */

/* <<< ilu_k_dm_create_adaptive(), DOF_MATRIX version */

/** Adaptively try to generate an ILU(k) preconditioner. Try to
 * factorize @f$A + \alpha I@f$. Additionally, off-diagonal elements
 * are weighted by a factor @f$\beta < 1@f$.
 *
 * @param[in]     A     The CRS-matrix to factorize.
 * @param[in,out] ilu   The ILU(k) fill structure as returned by
 *                      ilu_k_create_profile().
 * @param[in]     info  Noisiness.
 *
 * @return The number of iterations needed to adapt the off-diagonal
 *         weights and the diagonal offset.
 */
int ilu_k_dm_create_adaptive(const DOF_MATRIX *A, CRS_MATRIX *ilu, int info)
{
  FUNCNAME("ilu_k_dm_create_adaptive");
  REAL alpha = 0.0, beta = 1.0, coef = 1.0;
  int badness = 0;

  while (ilu_k_dm_create(A, ilu, alpha, beta, info) < 0) {
    coef  *= 2.0;
    alpha  = coef;
    /* beta   = 1.0/coef; */
    badness ++;
    INFO(info, INFO_VERBOSE,
	 "Using alpha = %e, beta = %e\n", alpha, beta);
  }
  return badness;
}

/* >>> */

/* >>> */

/* >>> */

/* <<< LU solvers. */

/** Solving is quite simple, because the rows of ilu are sorted in
 * ascending order and ilu->row[dof].col[0] holds the index of the
 * first upper diagonal element. Note that the column indices of the
 * precondtioner are already permuted.
 *
 * We have to solve @f$ P^{tr} LU P x = b <=> LU P x = P b @f$.
 *
 * We solve in-place. This version is for REAL vectors and scalar
 * matrices. See ilu_k_create().
 *
 * @param[in]     ilu  The incomplete factorization.
 * @param[in,out] x    The right hand side on input and the solution
 *                     on return from this function.
 */
static inline
void __ilu_k_solve(const CRS_MATRIX *ilu, REAL *x)
{
  int dof;
  CRS_OFFSET ioff;
  const CRS_MATRIX_INFO *info = ilu->info;
  int *PI = info->PI;

  /* solve (L + D)z = x in-place */
  for (dof = 0; dof < info->dim; dof ++) {
    DOF PIdof = PI[dof];
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_U(ilu, dof); ioff++) {
      x[PIdof] -= x[info->col[ioff]] * CRS_VALUE(ilu, ioff, REAL);
    }
    x[PIdof] *= CRS_DIAG(ilu, dof, REAL);
  }
  /* solve (1 + D^{-1}U) y = x in-place */
  for (dof = info->dim - 2; dof >= 0; dof--) {
    DOF PIdof = PI[dof];
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      x[PIdof] -= x[info->col[ioff]] * CRS_VALUE(ilu, ioff, REAL);
    }
  }
}
void ilu_k_solve(const CRS_MATRIX *ilu, REAL *x)
{
  __ilu_k_solve(ilu, x);
}

/** Solving is quite simple, because the rows of ilu are sorted in
 * ascending order and ilu->row[dof].col[0] holds the index of the
 * first upper diagonal element. Note that the column indices of the
 * precondtioner are already permuted.
 *
 * We have to solve @f$ P^{tr} LU P x = b <=> LU P x = P b @f$.
 *
 * We solve in-place. This version is for REAL_D vectors and scalar
 * matrices. See ilu_k_create().
 *
 * @param[in]     ilu  The incomplete factorization.
 * @param[in,out] x    The right hand side on input and the solution
 *                     on return from this function.
 */
static inline
void __ilu_k_solve_d(const CRS_MATRIX *ilu, REAL_D *x)
{
  int dof;
  CRS_OFFSET ioff;
  const CRS_MATRIX_INFO *info = ilu->info;
  int *PI = info->PI;

  /* solve (L + D)z = x in-place */
  for (dof = 0; dof < info->dim; dof ++) {
    REAL *entry = x[PI[dof]];
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_U(ilu, dof); ioff++) {
      AXPY_DOW(-CRS_VALUE(ilu, ioff, REAL), x[info->col[ioff]], entry);
    }
    SCAL_DOW(CRS_DIAG(ilu, dof, REAL), entry);
  }
  /* solve (1 + D^{-1}U) y = x in-place */
  for (dof = info->dim - 2; dof >= 0; dof--) {
    REAL *entry = x[PI[dof]];
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {
      AXPY_DOW(-CRS_VALUE(ilu, ioff, REAL), x[info->col[ioff]], entry);
    }
  }
}
void ilu_k_solve_d(const CRS_MATRIX *ilu, REAL_D *x)
{
  __ilu_k_solve_d(ilu, x);
}

/** Solving is quite simple, because the rows of ilu are sorted in
 * ascending order and ilu->row[dof].col[0] holds the index of the
 * first upper diagonal element. Note that the column indices of the
 * precondtioner are already permuted.
 *
 * We have to solve @f$ P^{tr} LU P x = b <=> LU P x = P b @f$.
 *
 * We solve in-place. This version is for REAL_D vectors and block
 * matrices. See ilu_k_create_dd().
 *
 * @param[in]     ilu  The incomplete factorization.
 * @param[in,out] x    The right hand side on input and the solution
 *                     on return from this function.
 */
static inline
void __ilu_k_solve_b(const CRS_MATRIX *ilu, REAL_D *x)
{
  int dof;
  CRS_OFFSET ioff;
  const CRS_MATRIX_INFO *info = ilu->info;
  int *PI = info->PI;

  /* solve (L + D)z = x in-place */
  for (dof = 0; dof < info->dim; dof ++) {
    DOF PIdof = PI[dof];
    REAL_D accu;
    COPY_DOW(x[PIdof], accu);
    for (ioff = CRS_BEG(ilu, dof) + 1; ioff < CRS_U(ilu, dof); ioff++) {
      MGEMV_DOW(-1.0, (const REAL_D *)CRS_VALUE(ilu, ioff, REAL_DD),
		x[info->col[ioff]],
		1.0, accu);
    }
    MVEQ_DOW((const REAL_D *)CRS_DIAG(ilu, dof, REAL_DD), accu, x[PIdof]);
  }
  /* solve (1 + D^{-1}U) y = x in-place */
  for (dof = info->dim - 2; dof >= 0; dof--) {
    REAL *entry = x[PI[dof]];
    for (ioff = CRS_U(ilu, dof); ioff < CRS_END(ilu, dof); ioff++) {      
      MGEMV_DOW(-1.0, (const REAL_D *)CRS_VALUE(ilu, ioff, REAL_DD),
		x[info->col[ioff]], 1.0, entry);
    }
  }
}
void ilu_k_solve_b(const CRS_MATRIX *ilu, REAL_D *x)
{
  __ilu_k_solve_b(ilu, x);
}

/* >>> */

/* <<< ALBERTA interface */

/* ALBERTA pre-conditioner interface */
struct ilu_k_precon_data 
{
  PRECON precon; /* must come first "base class" */
  
  const DOF_MATRIX    *matrix; /* original matrix */
  const DOF_SCHAR_VEC *mask;
  size_t              dim;

  CRS_MATRIX          *ilu; /* the preconditioner */

  int                 level, info;
};

/* Scalar matrix, scalar fe-space */
static void ILUk_precon_s(void *vdata, int dim, REAL *r)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;
  __ilu_k_solve(data->ilu, r);
}

/* Scalar matrix, DOW fe-space */
static void ILUk_precon_d(void *vdata, int dim, REAL *r)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;
  __ilu_k_solve_d(data->ilu, (REAL_D *)r);
}

/* DOWxDOW block-matrix, DOW fe-space */
static void ILUk_precon_dow(void *vdata, int dim, REAL *r)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;
  __ilu_k_solve_b(data->ilu, (REAL_D *)r);
}

static void exit_ILUk_precon(void *vdata)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;
  CRS_MATRIX_INFO *minfo = (CRS_MATRIX_INFO *)data->ilu->info;

  crs_matrix_free(data->ilu);
  if (dbl_list_empty(&minfo->matrices)) {
    crs_matrix_info_free(minfo);
  }

  MEM_FREE(data, 1, struct ilu_k_precon_data);
}

static bool init_ILUk_precon_s(void *vdata)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;

  ilu_k_dm_create_adaptive(data->matrix, data->ilu, data->info);

  return true;
}

static bool init_ILUk_precon_dow(void *vdata)
{
  struct ilu_k_precon_data *data = (struct ilu_k_precon_data *)vdata;

  ilu_k_dm_create_dd_adaptive(data->matrix, data->ilu, data->info);

  return true;
}

const PRECON *get_ILUk_precon(const DOF_MATRIX *A,
			      const DOF_SCHAR_VEC *mask,
			      int ilu_level, int info)
{
  struct ilu_k_precon_data *data;
  
  if (A->is_diagonal) {
    return get_diag_precon(A, mask); /* ... and already this is questionable */
  }

  data = MEM_CALLOC(1, struct ilu_k_precon_data);

  data->precon.precon_data = data;
  data->precon.exit_precon = exit_ILUk_precon;
  
  data->level  = ilu_level;
  data->info   = info;
  data->matrix = A;
  data->mask   = mask;
  data->ilu    = ilu_k_dm_create_profile(A, mask, ilu_level, info);

  switch (A->type) {
  case MATENT_REAL_DD:
    data->precon.init_precon = init_ILUk_precon_dow;
    data->precon.precon      = ILUk_precon_dow;
    break;
  case MATENT_REAL:
    data->precon.init_precon = init_ILUk_precon_s;
    if (A->row_fe_space->rdim == DIM_OF_WORLD
	&&
	A->row_fe_space->bas_fcts->rdim == 1) {
      data->precon.precon = ILUk_precon_d;
    } else {
      data->precon.precon = ILUk_precon_s;
    }
    break;
  default:
    ERROR_EXIT("Unsupported block-matrix type: %d\n", A->type);
    break;
  }

  return &data->precon;
}

/* >>> */

/**@}*/
/**@}*/

