/*--------------------------------------------------------------------------*/
/* ALBERTA:  an Adaptive multi Level finite element toolbox using           */
/*           Bisectioning refinement and Error control by Residual          */
/*           Techniques for scientific Applications                         */
/*                                                                          */
/* file:     quad_cache.c                                                   */
/*                                                                          */
/* description:  Fast (i.e. caching) quadrature for matrix assembling       */
/*                                                                          */
/*--------------------------------------------------------------------------*/
/*                                                                          */
/*  authors:   Alfred Schmidt                                               */
/*             Zentrum fuer Technomathematik                                */
/*             Fachbereich 3 Mathematik/Informatik                          */
/*             Universitaet Bremen                                          */
/*             Bibliothekstr. 2                                             */
/*             D-28359 Bremen, Germany                                      */
/*                                                                          */
/*             Kunibert G. Siebert                                          */
/*             Institut fuer Mathematik                                     */
/*             Universitaet Augsburg                                        */
/*             Universitaetsstr. 14                                         */
/*             D-86159 Augsburg, Germany                                    */
/*                                                                          */
/*             Claus-Justus Heine                                           */
/*             Abteilung fuer Angewandte Mathematik                         */
/*             Albert-Ludwigs-Universitaet Freiburg                         */
/*             Hermann-Herder-Str. 10                                       */
/*             D-79104 Freiburg im Breisgau, Germany                        */
/*                                                                          */
/*  http://www.alberta-fem.de                                               */
/*                                                                          */
/*  (c) by A. Schmidt and K.G. Siebert (1996-2003),                         */
/*         C.-J. Heine (2006-2009)                                          */
/*                                                                          */
/*--------------------------------------------------------------------------*/

#include "alberta_intern.h"
#include "alberta.h"

#if ALBERTA_DEBUG == 1
# define inline /* */
static const REAL TOO_SMALL = 10.0*REAL_EPSILON; /*SQR(REAL_EPSILON);*/
#else
# define TOO_SMALL (10.0*REAL_EPSILON) /* SQR(REAL_EPSILON) */
#endif

/*--------------------------------------------------------------------------*/
/* information about precomputed integrals of basis functions on the        */
/* standard element;                                                        */
/*--------------------------------------------------------------------------*/

#define PSI_PHI_MAGIC "AIPP"

/* NOTE: the code below relies on the fact that all of the quad-caches
 * have the following structure:
 */
typedef struct _AI_psi_phi
{
  const BAS_FCTS *psi;
  const BAS_FCTS *phi;
  const QUAD     *quad;

  const void     *cache;

  INIT_ELEMENT_DECL;

} _AI_PSI_PHI;

/* the meta information needed to support the INIT_ELEMENT() stuff,
 * and the list management.
 */
typedef struct _AI_meta_psi_phi _AI_META_PSI_PHI;
struct _AI_meta_psi_phi
{
  _AI_PSI_PHI       psi_phi;
  
  char              magic[4];
  _AI_META_PSI_PHI  *next;
  INIT_EL_TAG_CTX   tag_ctx;
  INIT_EL_TAG       psi_qf_tag;
  INIT_EL_TAG       phi_qf_tag;

  const QUAD_FAST   *psi_qf;
  const QUAD_FAST   *phi_qf;

  int               n_allocated_psi;
  int               n_allocated_phi;
  union {
    struct { int n_psi; int n_phi; } base;
    Q11_PSI_PHI_CACHE q11;
    Q11_PSI_PHI_CACHE q01;
    Q11_PSI_PHI_CACHE q10;
    Q11_PSI_PHI_CACHE q00;
  } cache;
  union {
    struct { int n_psi; int n_phi; } base;
    Q11_PSI_PHI_CACHE q11;
    Q11_PSI_PHI_CACHE q01;
    Q11_PSI_PHI_CACHE q10;
    Q11_PSI_PHI_CACHE q00;
  } dflt_cache;
};

static inline void realloc_psi_phi_11(void *_cache,
				      int n_mem_psi, int n_mem_phi,
				      int n_psi, int n_phi,
				      int dim)
{
  Q11_PSI_PHI_CACHE *cache = (Q11_PSI_PHI_CACHE *)_cache;
  const REAL **const*values;
  const int  **const*k, **const*l;

  if (cache->n_entries) {
    /* Free the old buffer */
    MAT_FREE(cache->n_entries,
	     n_mem_psi, n_mem_phi, int);
    ARRAY3_FREE(cache->values,
		n_mem_psi, n_mem_phi, SQR(N_LAMBDA(dim)), REAL);
    ARRAY3_FREE(cache->k,
		n_mem_psi, n_mem_phi, SQR(N_LAMBDA(dim)), int);
    ARRAY3_FREE(cache->l,
		n_mem_psi, n_mem_phi, SQR(N_LAMBDA(dim)), int);
  }

  /* Allocate the new buffers */
  cache->n_entries = (const int *const*)MAT_ALLOC(n_psi, n_phi, int);
  cache->values    = (const REAL *const*const*)MAT_ALLOC(n_psi, n_phi, REAL *);
  cache->k         = (const int *const*const*)MAT_ALLOC(n_psi, n_phi, int *);
  cache->l         = (const int *const*const*)MAT_ALLOC(n_psi, n_phi, int *);
  
  if (n_psi > 0 && n_phi > 0) {
    values = (const REAL **const*)cache->values;
    l      = (const int **const*)cache->l;
    k      = (const int **const*)cache->k;
  
    values[0][0] = MEM_ALLOC(n_psi*n_phi*SQR(N_LAMBDA(dim)), REAL);
    k[0][0]      = MEM_ALLOC(n_psi*n_phi*SQR(N_LAMBDA(dim)), int);
    l[0][0]      = MEM_ALLOC(n_psi*n_phi*SQR(N_LAMBDA(dim)), int);    
  }
}

static inline void compute_psi_phi_11(const QUAD_FAST *q_psi,
				      const QUAD_FAST *q_phi,
				      void *_cache)
{
  Q11_PSI_PHI_CACHE *cache = (Q11_PSI_PHI_CACHE *)_cache;
  int  *const*n_entries = (int *const*)cache->n_entries;
  REAL **const*values   = (REAL **const*)cache->values;
  int  **const*kk       = (int **const*)cache->k;
  int  **const*ll       = (int **const*)cache->l;
  const REAL_B *const*grd_psi;
  const REAL_B *const*grd_phi;
  const REAL   *w;
  int  dim = q_psi->dim;
  REAL *val_vec;
  int  *k_vec;
  int  *l_vec;
  int  i, j, k, l, n, iq;

  if (cache->n_psi == 0 || cache->n_phi == 0) {
    return;
  }

  val_vec = values[0][0];
  k_vec   = kk[0][0];
  l_vec   = ll[0][0];

  grd_phi = q_phi->grd_phi;
  grd_psi = q_psi->grd_phi;
  w       = q_psi->w;

  for (i = 0; i < cache->n_psi; i++) {
    for (j = 0; j < cache->n_phi; j++) {
      /* Compress the 3d arrays to be somewhat more cache friendly */
      values[i][j] = val_vec;
      kk[i][j] = k_vec;
      ll[i][j] = l_vec;

#if 0
      for (n = k = 0; k < N_LAMBDA(dim); k++) {
	for (l =  0; l < N_LAMBDA(dim); l++) {
	  REAL val = 0.0;
	  for (iq = 0; iq < q_psi->n_points; iq++) {
	    val += w[iq]*grd_psi[iq][i][k]*grd_phi[iq][j][l];
	  }
	  if (ABS(val) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val;
	    *k_vec++ = k;
	    *l_vec++ = l;
	  }
	}
      }
      n_entries[i][j] = n;
#else
      {
	REAL_BB val = { { 0.0, }, };
	for (iq = 0; iq < q_psi->n_points; iq++) {
	  for (k = 0; k < N_LAMBDA_MAX; k++) {
	    for (l = 0; l < N_LAMBDA_MAX; l++) {
	      val[k][l] += w[iq]*grd_psi[iq][i][k]*grd_phi[iq][j][l];
	    }
	  }
	}
	for (n = k = 0; k < N_LAMBDA(dim); k++) {
	  for (l =  0; l < N_LAMBDA(dim); l++) {
	    if (ABS(val[k][l]) > TOO_SMALL) {
	      n++;
	      *val_vec++ = val[k][l];
	      *k_vec++ = k;
	      *l_vec++ = l;
	    }
	  }
	}
	n_entries[i][j] = n;	
      }
#endif
    }
  }
}

static inline void realloc_psi_phi_01(void *_cache,
				      int n_mem_psi, int n_mem_phi,
				      int n_psi, int n_phi,
				      int dim)
{
  Q01_PSI_PHI_CACHE *cache = (Q01_PSI_PHI_CACHE *)_cache;
  const REAL **const*values;
  const int  **const*l;

  if (cache->n_entries) {
    /* Free the old buffer */
    MAT_FREE(cache->n_entries,
	     n_mem_psi, n_mem_phi, int);
    ARRAY3_FREE(cache->values, n_mem_psi, n_mem_phi, N_LAMBDA(dim), REAL);
    ARRAY3_FREE(cache->l, n_mem_psi, n_mem_phi, N_LAMBDA(dim), int);
  }

  /* Allocate the new buffers */
  cache->n_entries = (const int *const*)MAT_ALLOC(n_psi, n_phi, int);
  cache->values    = (const REAL *const*const*)MAT_ALLOC(n_psi, n_phi, REAL *);
  cache->l         = (const int *const*const*)MAT_ALLOC(n_psi, n_phi, int *);
  
  if (n_psi > 0 && n_phi > 0) {
    values = (const REAL **const*)cache->values;
    l      = (const int **const*)cache->l;
  
    values[0][0] = MEM_ALLOC(n_psi*n_phi*N_LAMBDA(dim), REAL);
    l[0][0]      = MEM_ALLOC(n_psi*n_phi*N_LAMBDA(dim), int);    
  }
}

static inline void compute_psi_phi_01(const QUAD_FAST *q_psi,
				      const QUAD_FAST *q_phi,
				      void *_cache)
{
  Q01_PSI_PHI_CACHE *cache = (Q01_PSI_PHI_CACHE *)_cache;
  int  *const*n_entries = (int *const*)cache->n_entries;
  REAL **const*values   = (REAL **const*)cache->values;
  int  **const*ll       = (int **const*)cache->l;
  int  dim              = q_psi->dim;
  REAL *val_vec;
  int  *l_vec;
  int  i, j, l, n, iq;
  const REAL   *w             = q_psi->w;
  const REAL   *const*psi     = q_psi->phi;
  const REAL_B *const*grd_phi = q_phi->grd_phi;
  int n_psi    = cache->n_psi;
  int n_phi    = cache->n_phi;
  int n_points = q_psi->n_points;

  if (n_psi == 0 || n_phi == 0) {
    return;
  }

  val_vec = values[0][0];
  l_vec   = ll[0][0];

  for (i = 0; i < n_psi; i++) {
    for (j = 0; j < n_phi; j++) {
      values[i][j] = val_vec;
      ll[i][j]     = l_vec;

#if 0
      for (n = l = 0; l < N_LAMBDA(dim); l++) {
	REAL val = 0.0;
	for (iq = 0; iq < n_points; iq++) {
	  REAL psii = psi[iq][i];
	  REAL grdj = grd_phi[iq][j][l];

	  val += w[iq]*psii*grdj;
	}
	if (ABS(val) > TOO_SMALL) {
	  n++;
	  *val_vec++ = val;
	  *l_vec++   = l;
	}
      }
      n_entries[i][j] = n;
#else
      {
	REAL_B val = { 0.0, };
	for (iq = 0; iq < n_points; iq++) {
	  AXPY_BAR(DIM_MAX, w[iq] * psi[iq][i], grd_phi[iq][j], val);
	}
	for (n = l = 0; l < N_LAMBDA(dim); l++) {
	  if (ABS(val[l]) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val[l];
	    *l_vec++   = l;
	  }
	}
	n_entries[i][j] = n;
      }
#endif
    }
  }
}

static inline void realloc_psi_phi_10(void *_cache,
				      int n_mem_psi, int n_mem_phi,
				      int n_psi, int n_phi,
				      int dim)
{
  Q10_PSI_PHI_CACHE *cache = (Q10_PSI_PHI_CACHE *)_cache;
  const REAL **const*values;
  const int  **const*k;

  if (cache->n_entries) {
    /* Free the old buffer */
    MAT_FREE(cache->n_entries,
	     n_mem_psi, n_mem_phi, int);
    ARRAY3_FREE(cache->values, n_mem_psi, n_mem_phi, N_LAMBDA(dim), REAL);
    ARRAY3_FREE(cache->k, n_mem_psi, n_mem_phi, N_LAMBDA(dim), int);
  }

  /* Allocate the new buffers */
  cache->n_entries = (const int *const*)MAT_ALLOC(n_psi, n_phi, int);
  cache->values    = (const REAL *const*const*)MAT_ALLOC(n_psi, n_phi, REAL *);
  cache->k         = (const int *const*const*)MAT_ALLOC(n_psi, n_phi, int *);
  
  if (n_psi > 0 && n_phi > 0) {
    values = (const REAL **const*)cache->values;
    k      = (const int **const*)cache->k;
  
    values[0][0] = MEM_ALLOC(n_psi*n_phi*N_LAMBDA(dim), REAL);
    k[0][0]      = MEM_ALLOC(n_psi*n_phi*N_LAMBDA(dim), int);
  }
}

static inline void compute_psi_phi_10(const QUAD_FAST *q_psi,
				      const QUAD_FAST *q_phi,
				      void *_cache)
{
  Q10_PSI_PHI_CACHE *cache = (Q10_PSI_PHI_CACHE *)_cache;
  int  *const*n_entries = (int *const*)cache->n_entries;
  REAL **const*values   = (REAL **const*)cache->values;
  int  **const*kk       = (int **const*)cache->k;
  int  dim              = q_psi->dim;
  REAL *val_vec;
  int  *k_vec;
  int  i, j, k, n, iq;
  const REAL   *w             = q_psi->w;
  const REAL_B *const*grd_psi = q_psi->grd_phi;
  const REAL   *const*phi     = q_phi->phi;
  int n_psi    = cache->n_psi;
  int n_phi    = cache->n_phi;
  int n_points = q_psi->n_points;

  if (n_psi == 0 || n_phi == 0) {
    return;
  }

  val_vec = values[0][0];
  k_vec   = kk[0][0];

  for (i = 0; i < cache->n_psi; i++) {
    for (j = 0; j < cache->n_phi; j++) {
      values[i][j] = val_vec;
      kk[i][j]     = k_vec;

#if 0
      for (n = k = 0; k < N_LAMBDA(dim); k++) {
	REAL val = 0.0;
	for (iq = 0; iq < n_points; iq++) {
	  REAL grdi = grd_psi[iq][i][k];
	  REAL phij = phi[iq][j];

	  val += w[iq]*grdi*phij;
	}
	if (ABS(val) > TOO_SMALL) {
	  n++;
	  *val_vec++ = val;
	  *k_vec++   = k;
	}
      }
      n_entries[i][j] = n;
#else
      {
	REAL_B val = { 0.0, };
	for (iq = 0; iq < n_points; iq++) {
	  AXPY_BAR(DIM_MAX, w[iq] * phi[iq][j], grd_psi[iq][i], val);
	}
	for (n = k = 0; k < N_LAMBDA(dim); k++) {
	  if (ABS(val[k]) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val[k];
	    *k_vec++   = k;
	  }
	}
	n_entries[i][j] = n;
      }
#endif
    }
  }
}

static inline void realloc_psi_phi_00(void *_cache,
				      int n_mem_psi, int n_mem_phi,
				      int n_psi, int n_phi,
				      int dim)
{
  Q00_PSI_PHI_CACHE *cache = (Q00_PSI_PHI_CACHE *)_cache;
  
  if (cache->values) {
    /* Free the old buffer */
    MAT_FREE(cache->values, n_mem_psi, n_mem_phi, REAL);
  }

  /* Allocate the new buffers */
  cache->values = (const REAL *const*)MAT_ALLOC(n_psi, n_phi, REAL);
}

static inline void
compute_psi_phi_00(const QUAD_FAST *q_psi,
		   const QUAD_FAST *q_phi,
		   void *_cache)
{
  Q00_PSI_PHI_CACHE *cache = (Q00_PSI_PHI_CACHE *)_cache;
  REAL **values   = (REAL **)cache->values;
  int  i, j, iq;
  REAL val;

  for (i = 0; i < cache->n_psi; i++) {
    for (j = 0; j < cache->n_phi; j++) {
      for (val = iq = 0; iq < q_psi->n_points; iq++)
	val += q_psi->w[iq]*q_psi->phi[iq][i]*q_phi->phi[iq][j];

      values[i][j] = val;
    }
  }
}

/* Define template like get...() and init_element stuff for all four
 * cases; there is no need to duplicate too much code here.
 */
static inline INIT_EL_TAG
psi_phi_init_element(const EL_INFO *el_info,
		     _AI_META_PSI_PHI *ai_psi_phi,
		     void (*compute_psi_phi)(const QUAD_FAST *q_psi,
					     const QUAD_FAST *q_phi,
					     void *cache),
		     void (*realloc_psi_phi)(void *cache,
					     int n_mem_psi, int n_mem_phi,
					     int n_psi, int n_phi,
					     int dim))
{
  _AI_PSI_PHI     *psi_phi = &ai_psi_phi->psi_phi;
  INIT_EL_TAG     psi_qf_tag, phi_qf_tag;
  int             n_psi, n_phi, n_mem_psi, n_mem_phi;

  psi_qf_tag = INIT_ELEMENT_SINGLE(el_info, ai_psi_phi->psi_qf);
  if (ai_psi_phi->psi_qf != ai_psi_phi->phi_qf) {
    phi_qf_tag = INIT_ELEMENT_SINGLE(el_info, ai_psi_phi->phi_qf);
  } else {
    phi_qf_tag = psi_qf_tag;
  }

  /* Check whether anything has changed */
  if (psi_qf_tag == ai_psi_phi->psi_qf_tag &&
      phi_qf_tag == ai_psi_phi->phi_qf_tag) {
    return INIT_EL_TAG_CTX_TAG(&ai_psi_phi->tag_ctx);
  }

  ai_psi_phi->psi_qf_tag = psi_qf_tag;
  ai_psi_phi->phi_qf_tag = phi_qf_tag;

  /* Possibly restore the default case. */
  if (psi_qf_tag == INIT_EL_TAG_DFLT && phi_qf_tag == INIT_EL_TAG_DFLT) {
    psi_phi->cache = &ai_psi_phi->dflt_cache;

    INIT_EL_TAG_CTX_DFLT(&ai_psi_phi->tag_ctx);

    return INIT_EL_TAG_CTX_TAG(&ai_psi_phi->tag_ctx);
  }

  /* Setup buffer pointers */
  psi_phi->cache = &ai_psi_phi->cache;
  
  /* Recompute cache values as needed. */
  if (psi_qf_tag != INIT_EL_TAG_NULL && phi_qf_tag != INIT_EL_TAG_NULL) {

    /* Check memory buffer size */
    n_psi     = psi_phi->psi->n_bas_fcts;
    n_phi     = psi_phi->phi->n_bas_fcts;
    n_mem_psi = ai_psi_phi->n_allocated_psi;
    n_mem_phi = ai_psi_phi->n_allocated_phi;
    
    if (n_mem_psi < n_psi || n_mem_phi < n_phi) {
      int n_alloc_psi = MAX(n_mem_psi,
			    MIN(2*psi_phi->psi->n_bas_fcts,
				psi_phi->psi->n_bas_fcts_max));
      int n_alloc_phi = MAX(n_mem_phi,
			    MIN(2*psi_phi->phi->n_bas_fcts,
				psi_phi->phi->n_bas_fcts_max));
      realloc_psi_phi(&ai_psi_phi->cache,
		      n_mem_psi, n_mem_phi, 
		      n_alloc_psi, n_alloc_phi,
		      psi_phi->psi->dim);
      ai_psi_phi->n_allocated_psi = n_alloc_psi;
      ai_psi_phi->n_allocated_phi = n_alloc_phi;
    }
  
    ai_psi_phi->cache.base.n_psi = n_psi;
    ai_psi_phi->cache.base.n_phi = n_phi;

    /* compute the new values */
    compute_psi_phi(ai_psi_phi->psi_qf,
		    ai_psi_phi->phi_qf, &ai_psi_phi->cache);
    INIT_EL_TAG_CTX_UNIQ(&ai_psi_phi->tag_ctx);
  } else {
    INIT_EL_TAG_CTX_NULL(&ai_psi_phi->tag_ctx);
  }

  return INIT_EL_TAG_CTX_TAG(&ai_psi_phi->tag_ctx);
}

static inline const _AI_PSI_PHI *
get_psi_phi(_AI_META_PSI_PHI **firstp,
	    const BAS_FCTS *psi,
	    const BAS_FCTS *phi,
	    const QUAD *quad,
	    int n_derivatives,
	    U_CHAR psi_init, U_CHAR phi_init,
	    void (*compute_psi_phi)(const QUAD_FAST *q_psi,
				    const QUAD_FAST *q_phi,
				    void *cache),
	    void (*realloc_psi_phi)(void *cache,
				    int n_mem_psi, int n_mem_phi,
				    int n_psi, int n_phi,
				    int dim),
	    INIT_ELEMENT_FCT init_element)
{
  FUNCNAME("get_psi_phi");
  _AI_META_PSI_PHI *ai_psi_phi;
  _AI_PSI_PHI      *psi_phi;
  const QUAD_FAST  *q_phi, *q_psi;
  int              dim;
  int              init_element_needed;

  TEST_EXIT(psi || phi, "Requesting quadrature cache for nothing?\n");

  if (!psi) psi = phi;
  if (!phi) phi = psi;

  TEST_EXIT(psi->dim == phi->dim,
	    "Support dimensions for phi and psi do not match!\n");
  dim = phi->dim;

  if (INIT_ELEMENT_NEEDED(psi) || INIT_ELEMENT_NEEDED(phi)) {
    /* reset to default values */
    INIT_OBJECT(psi);
    INIT_OBJECT(phi);
  }

  if (!quad) {
    quad = get_quadrature(dim,
			  psi->unchained->degree
			  +
			  phi->unchained->degree
			  -
			  n_derivatives);
  }

  init_element_needed =
    INIT_ELEMENT_NEEDED(psi) ||
    INIT_ELEMENT_NEEDED(phi) ||
    INIT_ELEMENT_NEEDED(quad);

/*--------------------------------------------------------------------------*/
/*  look for an existing entry in the list                                  */
/*--------------------------------------------------------------------------*/

  for (ai_psi_phi = *firstp; ai_psi_phi; ai_psi_phi = ai_psi_phi->next) {
    DEBUG_TEST_EXIT(memcmp(ai_psi_phi->magic,
			   PSI_PHI_MAGIC,
			   sizeof(ai_psi_phi->magic)) == 0,
	"Magic string was overwritten, data inconsistency, aborting.\n");
    psi_phi = &ai_psi_phi->psi_phi;
    if (psi_phi->psi == psi && psi_phi->phi == phi && psi_phi->quad == quad) {
      if (init_element_needed) {
	if (!INIT_ELEMENT_NEEDED(psi_phi)) {
	  ai_psi_phi->psi_qf_tag = ai_psi_phi->phi_qf_tag = INIT_EL_TAG_DFLT;
	  INIT_ELEMENT_DEFUN(psi_phi, init_element,
			     psi->fill_flags|phi->fill_flags|quad->fill_flags);

	  INIT_EL_TAG_CTX_INIT(&ai_psi_phi->tag_ctx);

	  ai_psi_phi->n_allocated_psi = 
	    ai_psi_phi->n_allocated_phi = 0; 
	}
	INIT_OBJECT(psi_phi);
      }
      return psi_phi;
    }
  }

/*--------------------------------------------------------------------------*/
/*  create a new one                                                        */
/*--------------------------------------------------------------------------*/

  ai_psi_phi       = MEM_CALLOC(1, _AI_META_PSI_PHI);
  memcpy(ai_psi_phi->magic, PSI_PHI_MAGIC, sizeof(ai_psi_phi->magic));
  ai_psi_phi->next = *firstp;
  *firstp          = ai_psi_phi;
  psi_phi          = &ai_psi_phi->psi_phi;

  ai_psi_phi->psi_qf = q_psi = get_quad_fast(psi, quad, psi_init);
  ai_psi_phi->phi_qf = q_phi = get_quad_fast(phi, quad, phi_init);

  realloc_psi_phi(&ai_psi_phi->dflt_cache,
		  0, 0, psi->n_bas_fcts, phi->n_bas_fcts, dim);

  ai_psi_phi->dflt_cache.base.n_psi = psi->n_bas_fcts;
  ai_psi_phi->dflt_cache.base.n_phi = phi->n_bas_fcts;  
  psi_phi->cache = &ai_psi_phi->dflt_cache;
  psi_phi->psi   = psi;
  psi_phi->phi   = phi;
  psi_phi->quad  = quad;

/*--------------------------------------------------------------------------*/
/* and now, fill information                                                */
/*--------------------------------------------------------------------------*/
  compute_psi_phi(q_psi, q_phi, &ai_psi_phi->dflt_cache);

  /* Setup up special init_element() data if needed. */
  if (init_element_needed) {
    ai_psi_phi->psi_qf_tag = ai_psi_phi->phi_qf_tag = INIT_EL_TAG_DFLT;
    INIT_ELEMENT_DEFUN(psi_phi, init_element,
		       psi->fill_flags|phi->fill_flags|quad->fill_flags);

    INIT_EL_TAG_CTX_INIT(&ai_psi_phi->tag_ctx);

    ai_psi_phi->n_allocated_psi = 
      ai_psi_phi->n_allocated_phi = 0;
  }

  return psi_phi;
}

/*--------------------------------------------------------------------------*/
/* second order term:                                                       */
/*--------------------------------------------------------------------------*/

FLATTEN_ATTR
static INIT_EL_TAG q11_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return psi_phi_init_element(el_info, (_AI_META_PSI_PHI *)thisptr,
			      compute_psi_phi_11,
			      realloc_psi_phi_11);
}

const Q11_PSI_PHI *get_q11_psi_phi(const BAS_FCTS *psi,
				   const BAS_FCTS *phi,
				   const QUAD *quad)
{
  static _AI_META_PSI_PHI *first;
  
  return (const Q11_PSI_PHI *)get_psi_phi(&first, psi, phi, quad, 2,
					  INIT_GRD_PHI, INIT_GRD_PHI,
					  compute_psi_phi_11,
					  realloc_psi_phi_11,
					  q11_psi_phi_init_element);
}

/*--------------------------------------------------------------------------*/
/* first order term                                                         */
/*--------------------------------------------------------------------------*/

FLATTEN_ATTR
static
INIT_EL_TAG q01_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return psi_phi_init_element(el_info, (_AI_META_PSI_PHI *)thisptr,
			      compute_psi_phi_01,
			      realloc_psi_phi_01);
}

const Q01_PSI_PHI *get_q01_psi_phi(const BAS_FCTS *psi,
				   const BAS_FCTS *phi,
				   const QUAD *quad)
{
  static _AI_META_PSI_PHI *first;
  
  return (const Q01_PSI_PHI *)get_psi_phi(&first, psi, phi, quad, 1,
					  INIT_PHI, INIT_GRD_PHI,
					  compute_psi_phi_01,
					  realloc_psi_phi_01,
					  q01_psi_phi_init_element);
}

FLATTEN_ATTR
static
INIT_EL_TAG q10_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return psi_phi_init_element(el_info, (_AI_META_PSI_PHI *)thisptr,
			      compute_psi_phi_10,
			      realloc_psi_phi_10);
}

const Q10_PSI_PHI *get_q10_psi_phi(const BAS_FCTS *psi,
				   const BAS_FCTS *phi,
				   const QUAD *quad)
{
  static _AI_META_PSI_PHI *first;
  
  return (const Q10_PSI_PHI *)get_psi_phi(&first, psi, phi, quad, 1,
					  INIT_GRD_PHI, INIT_PHI,
					  compute_psi_phi_10,
					  realloc_psi_phi_10,
					  q10_psi_phi_init_element);
}

/*--------------------------------------------------------------------------*/
/*  zero order term:                                                        */
/*--------------------------------------------------------------------------*/

FLATTEN_ATTR
static
INIT_EL_TAG q00_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return psi_phi_init_element(el_info, (_AI_META_PSI_PHI *)thisptr,
			      compute_psi_phi_00,
			      realloc_psi_phi_00);
}

const Q00_PSI_PHI *get_q00_psi_phi(const BAS_FCTS *psi,
				   const BAS_FCTS *phi,
				   const QUAD *quad)
{
  static _AI_META_PSI_PHI *first;
  
  return (const Q00_PSI_PHI *)get_psi_phi(&first, psi, phi, quad, 0,
					  INIT_PHI, INIT_PHI,
					  compute_psi_phi_00,
					  realloc_psi_phi_00,
					  q00_psi_phi_init_element);
}

/* Same as the stuff above, but for tri-linear forms */

#define ETA_PSI_PHI_MAGIC "APPE"

/* NOTE: the code below relies on the fact that all of the quad-caches
 * have the following structure:
 */
typedef struct _AI_eta_psi_phi
{
  const BAS_FCTS *eta;
  const BAS_FCTS *psi;
  const BAS_FCTS *phi;
  const QUAD     *quad;

  const void     *cache;

  INIT_ELEMENT_DECL;

} _AI_ETA_PSI_PHI;

/* the meta information needed to support the INIT_ELEMENT() stuff,
 * and the list management.
 */
typedef struct _AI_meta_eta_psi_phi _AI_META_ETA_PSI_PHI;
struct _AI_meta_eta_psi_phi
{
  _AI_ETA_PSI_PHI       eta_psi_phi;
  
  char                  magic[4];
  _AI_META_ETA_PSI_PHI *next;
  INIT_EL_TAG_CTX      tag_ctx;
  INIT_EL_TAG          eta_qf_tag;
  INIT_EL_TAG          psi_qf_tag;
  INIT_EL_TAG          phi_qf_tag;

  const QUAD_FAST      *eta_qf;
  const QUAD_FAST      *psi_qf;
  const QUAD_FAST      *phi_qf;

  int                  n_allocated_eta;
  int                  n_allocated_psi;
  int                  n_allocated_phi;
  union {
    struct { int n_eta; int n_psi; int n_phi; } base;
    Q001_ETA_PSI_PHI_CACHE q001;
    Q010_ETA_PSI_PHI_CACHE q010;
    Q100_ETA_PSI_PHI_CACHE q100;
  } cache;
  union {
    struct { int n_eta; int n_psi; int n_phi; } base;
    Q001_ETA_PSI_PHI_CACHE q001;
    Q010_ETA_PSI_PHI_CACHE q010;
    Q100_ETA_PSI_PHI_CACHE q100;
  } dflt_cache;
};

static inline
void realloc_eta_psi_phi_001(void *_cache,
			     int n_mem_eta, int n_mem_psi, int n_mem_phi,
			     int n_eta, int n_psi, int n_phi,
			     int dim)
{
  Q001_ETA_PSI_PHI_CACHE *cache = (Q001_ETA_PSI_PHI_CACHE *)_cache;
  const REAL **const*const*values;
  const int  **const*const*l;

  if (n_mem_eta > 0 && n_mem_psi > 0 && n_mem_phi > 0) {
    /* Free the old buffer */
    ARRAY3_FREE(cache->n_entries,
		n_mem_eta, n_mem_psi, n_mem_phi, int);
    ARRAY4_FREE(cache->values,
		n_mem_eta, n_mem_psi, n_mem_phi, N_LAMBDA(dim), REAL);
    ARRAY4_FREE(cache->l,
		n_mem_eta, n_mem_psi, n_mem_phi, N_LAMBDA(dim), int);
  }

  /* Allocate the new buffers */
  cache->n_entries =
    (const int *const*const*)ARRAY3_ALLOC(n_eta, n_psi, n_phi, int);
  cache->values    =
    (const REAL *const*const*const*)ARRAY3_ALLOC(n_eta, n_psi, n_phi, REAL *);
  cache->l         = 
    (const int *const*const*const*)ARRAY3_ALLOC(n_eta, n_psi, n_phi, int *);
  
  if (n_eta > 0 && n_psi > 0 && n_phi > 0) {
    values = (const REAL **const*const*)cache->values;
    l      = (const int **const*const*)cache->l;
  
    values[0][0][0] = MEM_ALLOC(n_eta*n_psi*n_phi*N_LAMBDA(dim), REAL);
    l[0][0][0]      = MEM_ALLOC(n_eta*n_psi*n_phi*N_LAMBDA(dim), int);    
  }
}

static inline void compute_eta_psi_phi_001(const QUAD_FAST *q_eta,
					   const QUAD_FAST *q_psi,
					   const QUAD_FAST *q_phi,
					   void *_cache)
{
  Q001_ETA_PSI_PHI_CACHE *cache = (Q001_ETA_PSI_PHI_CACHE *)_cache;
  int  *const*const*n_entries = (int *const*const*)cache->n_entries;
  REAL **const*const*values   = (REAL **const*const*)cache->values;
  int  **const*const*ll       = (int **const*const*)cache->l;
  int  dim                    = q_psi->dim;
  REAL *val_vec;
  int  *l_vec;
  int  i, j, k, l, n, iq;
  int n_eta     = cache->n_eta;
  int n_psi     = cache->n_psi;
  int n_phi     = cache->n_phi;
  int n_points  = q_eta->n_points;
  const REAL *w = q_eta->w;
  const REAL   *const*eta     = q_eta->phi;
  const REAL   *const*psi     = q_psi->phi;
  const REAL_B *const*grd_phi = q_phi->grd_phi;

  if (n_eta == 0 || n_psi == 0 || n_phi == 0) {
    return;
  }

  val_vec = values[0][0][0];
  l_vec   = ll[0][0][0];

  for (k = 0; k < cache->n_eta; k++) {
    for (i = 0; i < cache->n_psi; i++) {
      for (j = 0; j < cache->n_phi; j++) {
	values[k][i][j] = val_vec;
	ll[k][i][j]     = l_vec;

#if 0
	for (n = l = 0; l < N_LAMBDA(dim); l++) {
	  REAL val = 0.0;
	  for (iq = 0; iq < n_points; iq++) {
	    val += w[iq] * eta[iq][i] * psi[iq][j] * grd_phi[iq][k][l];
	  }
	  if (ABS(val) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val;
	    *l_vec++   = l;
	  }
	}
	n_entries[k][i][j] = n;
#else
	{
	  REAL_B val = { 0.0, };
	  for (iq = 0; iq < n_points; iq++) {
	    AXPY_BAR(DIM_MAX,
		     w[iq] * eta[iq][i] * psi[iq][j],
		     grd_phi[iq][k],
		     val);
	  }
	  for (n = l = 0; l < N_LAMBDA(dim); l++) {	    
	    if (ABS(val[l]) > TOO_SMALL) {
	      n++;
	      *val_vec++ = val[l];
	      *l_vec++   = l;
	    }
	  }
	  n_entries[k][i][j] = n;
	}
#endif
      }
    }
  }
}

static inline void compute_eta_psi_phi_010(const QUAD_FAST *q_eta,
					   const QUAD_FAST *q_psi,
					   const QUAD_FAST *q_phi,
					   void *_cache)
{
  Q010_ETA_PSI_PHI_CACHE *cache = (Q010_ETA_PSI_PHI_CACHE *)_cache;
  int  *const*const*n_entries = (int *const*const*)cache->n_entries;
  REAL **const*const*values   = (REAL **const*const*)cache->values;
  int  **const*const*ll       = (int **const*const*)cache->l;
  int  dim                    = q_psi->dim;
  REAL *val_vec;
  int  *l_vec;
  int  i, j, k, l, n, iq;
  int n_eta     = cache->n_eta;
  int n_psi     = cache->n_psi;
  int n_phi     = cache->n_phi;
  int n_points  = q_eta->n_points;
  const REAL *w = q_eta->w;
  const REAL   *const*eta     = q_eta->phi;
  const REAL_B *const*grd_psi = q_psi->grd_phi;
  const REAL   *const*phi     = q_phi->phi;

  if (n_eta == 0 || n_psi == 0 || n_phi == 0) {
    return;
  }

  val_vec = values[0][0][0];
  l_vec   = ll[0][0][0];

  for (k = 0; k < cache->n_eta; k++) {
    for (i = 0; i < cache->n_psi; i++) {
      for (j = 0; j < cache->n_phi; j++) {
	values[k][i][j] = val_vec;
	ll[k][i][j]     = l_vec;

#if 0
	for (n = l = 0; l < N_LAMBDA(dim); l++) {
	  REAL val = 0.0;
	  for (iq = 0; iq < n_points; iq++) {
	    val += w[iq] * eta[iq][k] * grd_psi[iq][i][l] * phi[iq][j];
	  }
	  if (ABS(val) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val;
	    *l_vec++   = l;
	  }
	}
	n_entries[k][i][j] = n;
#else
	{
	  REAL_B val = { 0.0, };
	  for (iq = 0; iq < n_points; iq++) {
	    AXPY_BAR(DIM_MAX,
		     w[iq] * eta[iq][k] * phi[iq][j], 
		     grd_psi[iq][i],
		     val);
	  }
	  for (n = l = 0; l < N_LAMBDA(dim); l++) {
	    if (ABS(val[l]) > TOO_SMALL) {
	      n++;
	      *val_vec++ = val[l];
	      *l_vec++   = l;
	    }
	  }
	  n_entries[k][i][j] = n;
	}
#endif
      }
    }
  }
}

static inline void compute_eta_psi_phi_100(const QUAD_FAST *q_eta,
					   const QUAD_FAST *q_psi,
					   const QUAD_FAST *q_phi,
					   void *_cache)
{
  Q100_ETA_PSI_PHI_CACHE *cache = (Q100_ETA_PSI_PHI_CACHE *)_cache;
  int  *const*const*n_entries = (int *const*const*)cache->n_entries;
  REAL **const*const*values   = (REAL **const*const*)cache->values;
  int  **const*const*ll       = (int **const*const*)cache->l;
  int  dim                    = q_psi->dim;
  REAL *val_vec;
  int  *l_vec;
  int  i, j, k, l, n, iq;
  int n_eta     = cache->n_eta;
  int n_psi     = cache->n_psi;
  int n_phi     = cache->n_phi;
  int n_points  = q_eta->n_points;
  const REAL *w = q_eta->w;
  const REAL_B *const*grd_eta = q_eta->grd_phi;
  const REAL   *const*psi     = q_psi->phi;
  const REAL   *const*phi     = q_phi->phi;

  if (n_eta == 0 || n_psi == 0 || n_phi == 0) {
    return;
  }

  val_vec = values[0][0][0];
  l_vec   = ll[0][0][0];

  for (k = 0; k < n_eta; k++) {
    for (i = 0; i < n_psi; i++) {
      for (j = 0; j < n_phi; j++) {
	values[k][i][j] = val_vec;
	ll[k][i][j]     = l_vec;
#if 0
	for (n = l = 0; l < N_LAMBDA(dim); l++) {
	  REAL val = 0.0;
	  for (iq = 0; iq < n_points; iq++) {
	    val += w[iq] * grd_eta[iq][k][l] * psi[iq][i] * phi[iq][j];
	  }
	  if (ABS(val) > TOO_SMALL) {
	    n++;
	    *val_vec++ = val;
	    *l_vec++   = l;
	  }
	}
	n_entries[k][i][j] = n;
#else
	{
	  REAL_B val = { 0.0, };
	  for (iq = 0; iq < n_points; iq++) {
	    AXPY_BAR(DIM_MAX,
		     w[iq] * psi[iq][i] * phi[iq][j],
		     grd_eta[iq][k],
		     val);
	  }
	  for (n = l = 0; l < N_LAMBDA(dim); l++) {	    
	    if (ABS(val[l]) > TOO_SMALL) {
	      n++;
	      *val_vec++ = val[l];
	      *l_vec++   = l;
	    }
	  }
	  n_entries[k][i][j] = n;
	}
#endif
      }
    }
  }
}

/* Define template like get...() and init_element stuff for all four
 * cases; there is no need to duplicate too much code here.
 */
static inline INIT_EL_TAG
eta_psi_phi_init_element(const EL_INFO *el_info,
			 _AI_META_ETA_PSI_PHI *ai_eta_psi_phi,
			 void (*compute_eta_psi_phi)(const QUAD_FAST *q_eta,
						     const QUAD_FAST *q_psi,
						     const QUAD_FAST *q_phi,
						     void *cache),
			 void (*realloc_eta_psi_phi)(void *cache,
						     int n_mem_eta,
						     int n_mem_psi,
						     int n_mem_phi,
						     int n_eta,
						     int n_psi,
						     int n_phi,
						     int dim))
{
  _AI_ETA_PSI_PHI *eta_psi_phi = &ai_eta_psi_phi->eta_psi_phi;
  INIT_EL_TAG     psi_qf_tag, phi_qf_tag, eta_qf_tag;
  int             n_psi, n_phi, n_eta, n_mem_psi, n_mem_phi, n_mem_eta;

  psi_qf_tag = INIT_ELEMENT_SINGLE(el_info, ai_eta_psi_phi->psi_qf);
  if (ai_eta_psi_phi->phi_qf != ai_eta_psi_phi->psi_qf) {
    phi_qf_tag = INIT_ELEMENT_SINGLE(el_info, ai_eta_psi_phi->phi_qf);
  } else {
    phi_qf_tag = psi_qf_tag;
  }
  if (ai_eta_psi_phi->eta_qf != ai_eta_psi_phi->psi_qf &&
      ai_eta_psi_phi->eta_qf != ai_eta_psi_phi->phi_qf) {
    eta_qf_tag = INIT_ELEMENT_SINGLE(el_info, ai_eta_psi_phi->eta_qf);
  } else {
    eta_qf_tag =
      ai_eta_psi_phi->eta_qf == ai_eta_psi_phi->psi_qf
      ? psi_qf_tag : phi_qf_tag;
  }

  /* Check whether anything has changed */
  if (eta_qf_tag == ai_eta_psi_phi->eta_qf_tag &&
      psi_qf_tag == ai_eta_psi_phi->psi_qf_tag &&
      phi_qf_tag == ai_eta_psi_phi->phi_qf_tag) {
    return INIT_EL_TAG_CTX_TAG(&ai_eta_psi_phi->tag_ctx);
  }

  ai_eta_psi_phi->eta_qf_tag = eta_qf_tag;
  ai_eta_psi_phi->psi_qf_tag = psi_qf_tag;
  ai_eta_psi_phi->phi_qf_tag = phi_qf_tag;

  /* Possibly restore the default case. */
  if (eta_qf_tag == INIT_EL_TAG_DFLT &&
      psi_qf_tag == INIT_EL_TAG_DFLT &&
      phi_qf_tag == INIT_EL_TAG_DFLT) {
    eta_psi_phi->cache = &ai_eta_psi_phi->dflt_cache;

    INIT_EL_TAG_CTX_DFLT(&ai_eta_psi_phi->tag_ctx);

    return INIT_EL_TAG_CTX_TAG(&ai_eta_psi_phi->tag_ctx);
  }

  /* Setup buffer pointers */
  eta_psi_phi->cache = &ai_eta_psi_phi->cache;
  
  /* Recompute cache values as needed. */
  if (eta_qf_tag != INIT_EL_TAG_NULL &&
      psi_qf_tag != INIT_EL_TAG_NULL &&
      phi_qf_tag != INIT_EL_TAG_NULL) {

    /* Check memory buffer size */
    n_eta     = eta_psi_phi->eta->n_bas_fcts;
    n_psi     = eta_psi_phi->psi->n_bas_fcts;
    n_phi     = eta_psi_phi->phi->n_bas_fcts;
    n_mem_eta = ai_eta_psi_phi->n_allocated_eta;
    n_mem_psi = ai_eta_psi_phi->n_allocated_psi;
    n_mem_phi = ai_eta_psi_phi->n_allocated_phi;
    
    if (n_mem_eta < n_eta || n_mem_psi < n_psi || n_mem_phi < n_phi) {
      int n_alloc_eta = MAX(n_mem_eta,
			    MIN(2*eta_psi_phi->eta->n_bas_fcts,
				eta_psi_phi->eta->n_bas_fcts_max));
      int n_alloc_psi = MAX(n_mem_psi,
			    MIN(2*eta_psi_phi->psi->n_bas_fcts,
				eta_psi_phi->psi->n_bas_fcts_max));
      int n_alloc_phi = MAX(n_mem_phi,
			    MIN(eta_psi_phi->phi->n_bas_fcts,
				eta_psi_phi->phi->n_bas_fcts_max));
      realloc_eta_psi_phi(&ai_eta_psi_phi->cache,
			  n_mem_eta, n_mem_psi, n_mem_phi,
			  n_alloc_eta, n_alloc_psi, n_alloc_phi,
			  eta_psi_phi->psi->dim);
      ai_eta_psi_phi->n_allocated_eta = n_alloc_eta;
      ai_eta_psi_phi->n_allocated_psi = n_alloc_psi;
      ai_eta_psi_phi->n_allocated_phi = n_alloc_phi;
    }
  
    ai_eta_psi_phi->cache.base.n_eta = n_eta;
    ai_eta_psi_phi->cache.base.n_psi = n_psi;
    ai_eta_psi_phi->cache.base.n_phi = n_phi;

    /* compute the new values */
    compute_eta_psi_phi(ai_eta_psi_phi->eta_qf,
			ai_eta_psi_phi->psi_qf,
			ai_eta_psi_phi->phi_qf,
			&ai_eta_psi_phi->cache);
    INIT_EL_TAG_CTX_UNIQ(&ai_eta_psi_phi->tag_ctx);
  } else {
    INIT_EL_TAG_CTX_NULL(&ai_eta_psi_phi->tag_ctx);
  }

  return INIT_EL_TAG_CTX_TAG(&ai_eta_psi_phi->tag_ctx);
}

static inline const _AI_ETA_PSI_PHI *
get_eta_psi_phi(_AI_META_ETA_PSI_PHI **firstp,
		const BAS_FCTS *eta,
		const BAS_FCTS *psi,
		const BAS_FCTS *phi,
		const QUAD *quad,
		int n_derivatives,
		U_CHAR eta_init, U_CHAR psi_init, U_CHAR phi_init,
		void (*compute_eta_psi_phi)(const QUAD_FAST *q_eta,
					    const QUAD_FAST *q_psi,
					    const QUAD_FAST *q_phi,
					    void *cache),
		void (*realloc_eta_psi_phi)(void *cache,
					    int n_mem_eta,
					    int n_mem_psi,
					    int n_mem_phi,
					    int n_eta,
					    int n_psi,
					    int n_phi,
					    int dim),
		INIT_ELEMENT_FCT init_element)
{
  FUNCNAME("get_eta_psi_phi");
  _AI_META_ETA_PSI_PHI *ai_eta_psi_phi;
  _AI_ETA_PSI_PHI      *eta_psi_phi;
  const QUAD_FAST      *q_eta, *q_phi, *q_psi;
  int                  dim;
  int                  init_element_needed;

  TEST_EXIT(eta || psi || phi, "Requesting quadrature cache for nothing?\n");

  if (psi == NULL) {
    psi = phi;
    if (psi == NULL) {
      psi = eta;
    }
  }
  if (phi == NULL) {
    phi = psi;
    if (phi == NULL) {
      phi = eta;
    }
  }
  if (eta == NULL) {
    eta = psi;
    if (eta == NULL) {
      eta = phi;
    }
  }

  TEST_EXIT(psi->dim == phi->dim && psi->dim == eta->dim,
	    "Support dimensions for phi and psi do not match!\n");
  dim = psi->dim;

  if (INIT_ELEMENT_NEEDED(eta) ||
      INIT_ELEMENT_NEEDED(psi) ||
      INIT_ELEMENT_NEEDED(phi)) {
    /* reset to default values */
    INIT_OBJECT(eta);
    INIT_OBJECT(psi);
    INIT_OBJECT(phi);
  }

  if (!quad) {
    quad = get_quadrature(dim,
			  eta->unchained->degree
			  +
			  psi->unchained->degree
			  +
			  phi->unchained->degree
			  -
			  n_derivatives);
  }

  init_element_needed =
    INIT_ELEMENT_NEEDED(eta) ||
    INIT_ELEMENT_NEEDED(psi) ||
    INIT_ELEMENT_NEEDED(phi) ||
    INIT_ELEMENT_NEEDED(quad);

/*--------------------------------------------------------------------------*/
/*  look for an existing entry in the list                                  */
/*--------------------------------------------------------------------------*/

  for (ai_eta_psi_phi = *firstp;
       ai_eta_psi_phi;
       ai_eta_psi_phi = ai_eta_psi_phi->next) {
    DEBUG_TEST_EXIT(memcmp(ai_eta_psi_phi->magic,
			   ETA_PSI_PHI_MAGIC,
			   sizeof(ai_eta_psi_phi->magic)) == 0,
		    "Magic string was overwritten, "
		    "data inconsistency, aborting.\n");
    eta_psi_phi = &ai_eta_psi_phi->eta_psi_phi;
    if (eta_psi_phi->eta == eta &&
	eta_psi_phi->psi == psi &&
	eta_psi_phi->phi == phi &&
	eta_psi_phi->quad == quad) {
      if (init_element_needed) {
	if (!INIT_ELEMENT_NEEDED(eta_psi_phi)) {
	  ai_eta_psi_phi->eta_qf_tag = 
	    ai_eta_psi_phi->psi_qf_tag =
	    ai_eta_psi_phi->phi_qf_tag = INIT_EL_TAG_DFLT;
	  INIT_ELEMENT_DEFUN(
	    eta_psi_phi, init_element,
	    eta->fill_flags|psi->fill_flags|phi->fill_flags|quad->fill_flags);

	  INIT_EL_TAG_CTX_INIT(&ai_eta_psi_phi->tag_ctx);
	  
	  ai_eta_psi_phi->n_allocated_eta =
	    ai_eta_psi_phi->n_allocated_psi = 
	    ai_eta_psi_phi->n_allocated_phi = 0;
	}
	INIT_OBJECT(eta_psi_phi);
      }
      return eta_psi_phi;
    }
  }

/*--------------------------------------------------------------------------*/
/*  create a new one                                                        */
/*--------------------------------------------------------------------------*/

  ai_eta_psi_phi       = MEM_CALLOC(1, _AI_META_ETA_PSI_PHI);
  memcpy(ai_eta_psi_phi->magic,
	 ETA_PSI_PHI_MAGIC, sizeof(ai_eta_psi_phi->magic));
  ai_eta_psi_phi->next = *firstp;
  *firstp              = ai_eta_psi_phi;
  eta_psi_phi          = &ai_eta_psi_phi->eta_psi_phi;

  ai_eta_psi_phi->eta_qf = q_eta = get_quad_fast(eta, quad, eta_init);
  ai_eta_psi_phi->psi_qf = q_psi = get_quad_fast(psi, quad, psi_init);
  ai_eta_psi_phi->phi_qf = q_phi = get_quad_fast(phi, quad, phi_init);

  realloc_eta_psi_phi(&ai_eta_psi_phi->dflt_cache,
		      0, 0, 0,
		      eta->n_bas_fcts, psi->n_bas_fcts, phi->n_bas_fcts,
		      dim);

  ai_eta_psi_phi->dflt_cache.base.n_eta = eta->n_bas_fcts;  
  ai_eta_psi_phi->dflt_cache.base.n_psi = psi->n_bas_fcts;
  ai_eta_psi_phi->dflt_cache.base.n_phi = phi->n_bas_fcts;  
  eta_psi_phi->cache = &ai_eta_psi_phi->dflt_cache;
  eta_psi_phi->eta   = eta;
  eta_psi_phi->psi   = psi;
  eta_psi_phi->phi   = phi;
  eta_psi_phi->quad  = quad;

  /* setup the default cache */
  compute_eta_psi_phi(q_eta, q_psi, q_phi, &ai_eta_psi_phi->dflt_cache);

  /* Setup up special init_element() data if needed. */
  if (init_element_needed) {
    ai_eta_psi_phi->psi_qf_tag = ai_eta_psi_phi->phi_qf_tag = INIT_EL_TAG_DFLT;
    INIT_ELEMENT_DEFUN(
      eta_psi_phi, init_element,
      eta->fill_flags|psi->fill_flags|phi->fill_flags|quad->fill_flags);

    INIT_EL_TAG_CTX_INIT(&ai_eta_psi_phi->tag_ctx);

    ai_eta_psi_phi->n_allocated_eta =
      ai_eta_psi_phi->n_allocated_psi = 
      ai_eta_psi_phi->n_allocated_phi = 0;
  }

  return eta_psi_phi;
}

FLATTEN_ATTR
static
INIT_EL_TAG q001_eta_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return eta_psi_phi_init_element(el_info, (_AI_META_ETA_PSI_PHI *)thisptr,
				  compute_eta_psi_phi_001,
				  realloc_eta_psi_phi_001);
}

const Q001_ETA_PSI_PHI *get_q001_eta_psi_phi(const BAS_FCTS *eta,
					     const BAS_FCTS *psi,
					     const BAS_FCTS *phi,
					     const QUAD *quad)
{
  static _AI_META_ETA_PSI_PHI *first;
  
  return (const Q001_ETA_PSI_PHI *)
    get_eta_psi_phi(&first, eta, psi, phi, quad, 1,
		    INIT_PHI, INIT_PHI, INIT_GRD_PHI,
		    compute_eta_psi_phi_001,
		    realloc_eta_psi_phi_001,
		    q001_eta_psi_phi_init_element);
}

FLATTEN_ATTR
static
INIT_EL_TAG q010_eta_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return eta_psi_phi_init_element(el_info, (_AI_META_ETA_PSI_PHI *)thisptr,
				  compute_eta_psi_phi_010,
				  realloc_eta_psi_phi_001);
}

const Q010_ETA_PSI_PHI *get_q010_eta_psi_phi(const BAS_FCTS *eta,
					     const BAS_FCTS *psi,
					     const BAS_FCTS *phi,
					     const QUAD *quad)
{
  static _AI_META_ETA_PSI_PHI *first;
  
  return (const Q010_ETA_PSI_PHI *)
    get_eta_psi_phi(&first, eta, psi, phi, quad, 1,
		    INIT_PHI, INIT_GRD_PHI, INIT_PHI,
		    compute_eta_psi_phi_010,
		    realloc_eta_psi_phi_001,
		    q010_eta_psi_phi_init_element);
}

FLATTEN_ATTR
static
INIT_EL_TAG q100_eta_psi_phi_init_element(const EL_INFO *el_info, void *thisptr)
{
  return eta_psi_phi_init_element(el_info, (_AI_META_ETA_PSI_PHI *)thisptr,
				  compute_eta_psi_phi_100,
				  realloc_eta_psi_phi_001);
}

const Q100_ETA_PSI_PHI *get_q100_eta_psi_phi(const BAS_FCTS *eta,
					     const BAS_FCTS *psi,
					     const BAS_FCTS *phi,
					     const QUAD *quad)
{
  static _AI_META_ETA_PSI_PHI *first;
  
  return (const Q100_ETA_PSI_PHI *)
    get_eta_psi_phi(&first, eta, psi, phi, quad, 1,
		    INIT_GRD_PHI, INIT_PHI, INIT_PHI,
		    compute_eta_psi_phi_100,
		    realloc_eta_psi_phi_001,
		    q100_eta_psi_phi_init_element);
}

