/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
 * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
 * Copyright (c) 2016      Mellanox Technologies, Inc.
 *                         All rights reserved.
 * Copyright (c) 2016      IBM Corporation.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 */
#include "src/include/pmix_config.h"

#include "include/pmix.h"
#include "include/pmix_common.h"
#include "include/pmix_server.h"

#include "src/mca/bfrops/bfrops.h"
#include "src/mca/gds/base/base.h"
#include "src/include/pmix_globals.h"
#include "src/threads/threads.h"
#include "src/client/pmix_client_ops.h"

#include "src/common/pmix_attributes.h"
#include "src/include/dictionary.h"

static bool initialized = false;
static pmix_list_t client_attrs;
static pmix_list_t server_attrs;
static pmix_list_t host_attrs;
static pmix_list_t tool_attrs;

typedef struct {
    char *function;
    char **attrs;
} pmix_attr_init_t;

typedef struct {
    pmix_list_item_t super;
    char *function;
    char **attrs;
} pmix_attribute_trk_t;

static void atrkcon(pmix_attribute_trk_t *p)
{
    p->function = NULL;
    p->attrs = NULL;
}
static void atrkdes(pmix_attribute_trk_t *p)
{
    if (NULL != p->function) {
        free(p->function);
    }
    pmix_argv_free(p->attrs);
}
static PMIX_CLASS_INSTANCE(pmix_attribute_trk_t,
                           pmix_list_item_t,
                           atrkcon, atrkdes);

PMIX_EXPORT void pmix_init_registered_attrs(void)
{
    if (!initialized) {
        PMIX_CONSTRUCT(&client_attrs, pmix_list_t);
        PMIX_CONSTRUCT(&server_attrs, pmix_list_t);
        PMIX_CONSTRUCT(&host_attrs, pmix_list_t);
        PMIX_CONSTRUCT(&tool_attrs, pmix_list_t);
        initialized = true;
    }
}

static pmix_status_t process_reg(char *level, char *function,
                                 char **attrs)
{
    pmix_attribute_trk_t *fnptr;
    pmix_list_t *lst;

    /* select the list this will appear on */
    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
        lst = &client_attrs;
    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
        lst = &server_attrs;
    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
        lst = &host_attrs;
    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
        lst = &tool_attrs;
    } else {
        return PMIX_ERR_BAD_PARAM;
    }

    /* see if we already have this function */
    PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
        if (0 == strcmp(function, fnptr->function)) {
            /* we already have this function at this level
             * so we must return an error */
            return PMIX_ERR_REPEAT_ATTR_REGISTRATION;
        }
    }

    fnptr = PMIX_NEW(pmix_attribute_trk_t);
    pmix_list_append(lst, &fnptr->super);
    fnptr->function = strdup(function);
    if (NULL != attrs) {
        fnptr->attrs = pmix_argv_copy(attrs);
    }
    return PMIX_SUCCESS;
}

PMIX_EXPORT pmix_status_t PMIx_Register_attributes(char *function, char *attrs[])
{
    pmix_status_t rc;

    PMIX_ACQUIRE_THREAD(&pmix_global_lock);
    if (pmix_globals.init_cntr <= 0) {
        PMIX_RELEASE_THREAD(&pmix_global_lock);
        return PMIX_ERR_INIT;
    }

    rc = process_reg(PMIX_HOST_ATTRIBUTES, function, attrs);
    PMIX_RELEASE_THREAD(&pmix_global_lock);
    return rc;
}

PMIX_EXPORT void pmix_release_registered_attrs(void)
{
    if (initialized) {
        PMIX_LIST_DESTRUCT(&client_attrs);
        PMIX_LIST_DESTRUCT(&server_attrs);
        PMIX_LIST_DESTRUCT(&host_attrs);
        PMIX_LIST_DESTRUCT(&tool_attrs);
    }
    initialized = false;
}

/* sadly, we cannot dynamically register our supported attributes
 * as that would require the user to first call the function whose
 * attributes they want to know about - which somewhat defeats the
 * purpose. Until someone comes up with a better solution, we will
 * manually maintain the list */
static pmix_attr_init_t client_fns[] = {
    {.function = "PMIx_Init", .attrs = (char *[]){"PMIX_GDS_MODULE", "PMIX_EVENT_BASE", "PMIX_HOSTNAME",
                                                  "PMIX_NODEID", "PMIX_DEBUG_STOP_IN_INIT", NULL}},
    {.function = "PMIx_Finalize", .attrs = (char *[]){"PMIX_EMBED_BARRIER", NULL}},
    {.function = "PMIx_Initialized", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Abort", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Store_internal", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Put", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Commit", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Fence", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Fence_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Get", .attrs = (char *[]){"PMIX_NODE_INFO", "PMIX_HOSTNAME", "PMIX_NODEID",
                                                 "PMIX_APP_INFO", "PMIX_APPNUM", "PMIX_SESSION_INFO",
                                                 "PMIX_GET_REFRESH_CACHE", NULL}},
    {.function = "PMIx_Get_nb", .attrs = (char *[]){"PMIX_NODE_INFO", "PMIX_HOSTNAME", "PMIX_NODEID",
                                                    "PMIX_APP_INFO", "PMIX_APPNUM", "PMIX_SESSION_INFO",
                                                    "PMIX_GET_REFRESH_CACHE", NULL}},
    {.function = "PMIx_Publish", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Publish_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Lookup", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Lookup_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Unpublish", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Unpublish_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Spawn", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", NULL}},
    {.function = "PMIx_Spawn_nb", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", NULL}},
    {.function = "PMIx_Connect", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Connect_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Disconnect", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Disconnect_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Resolve_peers", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Resolve_nodes", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Query_info", .attrs = (char *[]){"PMIX_QUERY_SUPPORTED_KEYS", "PMIX_QUERY_NAMESPACES",
                                                        "PMIX_QUERY_ATTRIBUTE_SUPPORT", "PMIX_QUERY_AVAIL_SERVERS",
                                                        "PMIX_QUERY_REFRESH_CACHE", "PMIX_QUERY_SUPPORTED_KEYS",
                                                        "PMIX_QUERY_SUPPORTED_QUALIFIERS", "PMIX_TIME_REMAINING", NULL}},
    {.function = "PMIx_Query_info_nb", .attrs = (char *[]){"PMIX_QUERY_SUPPORTED_KEYS", "PMIX_QUERY_NAMESPACES",
                                                           "PMIX_QUERY_ATTRIBUTE_SUPPORT", "PMIX_QUERY_AVAIL_SERVERS",
                                                           "PMIX_QUERY_REFRESH_CACHE", "PMIX_QUERY_SUPPORTED_KEYS",
                                                           "PMIX_QUERY_SUPPORTED_QUALIFIERS", "PMIX_TIME_REMAINING", NULL}},
    {.function = "PMIx_Log", .attrs = (char *[]){"PMIX_LOG_GENERATE_TIMESTAMP", "PMIX_LOG_SOURCE", NULL}},
    {.function = "PMIx_Log_nb", .attrs = (char *[]){"PMIX_LOG_GENERATE_TIMESTAMP", "PMIX_LOG_SOURCE", NULL}},
    {.function = "PMIx_Allocation_request", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Allocation_request_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Job_control", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Job_control_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Process_monitor", .attrs = (char *[]){"PMIX_SEND_HEARTBEAT", NULL}},
    {.function = "PMIx_Process_monitor_nb", .attrs = (char *[]){"PMIX_SEND_HEARTBEAT", NULL}},
    {.function = "PMIx_Get_credential", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
    {.function = "PMIx_Get_credential_nb", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
    {.function = "PMIx_Validate_credential", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
    {.function = "PMIx_Validate_credential_nb", .attrs = (char *[]){"PMIX_CRED_TYPE", NULL}},
    {.function = "PMIx_Group_construct", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_construct_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_invite", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_invite_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_join", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_join_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_leave", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_leave_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_destruct", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Group_destruct_nb", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Register_event_handler", .attrs = (char *[]){"PMIX_EVENT_HDLR_FIRST", "PMIX_EVENT_HDLR_LAST",
                                                                    "PMIX_EVENT_HDLR_PREPEND", "PMIX_EVENT_HDLR_APPEND",
                                                                    "PMIX_EVENT_HDLR_NAME", "PMIX_EVENT_RETURN_OBJECT",
                                                                    "PMIX_EVENT_HDLR_FIRST_IN_CATEGORY",
                                                                    "PMIX_EVENT_HDLR_LAST_IN_CATEGORY",
                                                                    "PMIX_EVENT_HDLR_BEFORE", "PMIX_EVENT_HDLR_AFTER",
                                                                    "PMIX_RANGE", "PMIX_EVENT_CUSTOM_RANGE",
                                                                    "PMIX_EVENT_AFFECTED_PROC", "PMIX_EVENT_AFFECTED_PROCS",
                                                                     NULL}},
    {.function = "PMIx_Deregister_event_handler", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Notify_event", .attrs = (char *[]){"PMIX_EVENT_NON_DEFAULT", "PMIX_EVENT_CUSTOM_RANGE",
                                                          "PMIX_EVENT_AFFECTED_PROC", "PMIX_EVENT_AFFECTED_PROCS", NULL}},
    {.function = "PMIx_Error_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Proc_state_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Scope_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Persistence_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_range_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Info_directives_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_type_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Alloc_directive_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_IOF_channel_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Job_state_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Get_attribute_string", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Get_attribute_name", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Get_version", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_pack", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_unpack", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_copy", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_print", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_Data_copy_payload", .attrs = (char *[]){"N/A", NULL}},
    {.function = ""}
};

/*****    REGISTER CLIENT ATTRS    *****/
static bool client_attrs_regd = false;

PMIX_EXPORT pmix_status_t pmix_register_client_attrs(void)
{
    size_t n;
    pmix_status_t rc = PMIX_SUCCESS;

    if (client_attrs_regd) {
        return PMIX_SUCCESS;
    }
    client_attrs_regd = true;

    for (n=0; 0 != strlen(client_fns[n].function); n++) {
        rc = process_reg(PMIX_CLIENT_ATTRIBUTES,
                         client_fns[n].function, client_fns[n].attrs);
        if (PMIX_SUCCESS != rc) {
            break;
        }
    }

    return rc;
}

static pmix_attr_init_t server_fns[] = {
    {.function = "PMIx_server_init", .attrs = (char *[]){"PMIX_SERVER_GATEWAY", "PMIX_SERVER_SCHEDULER",
                                                         "PMIX_SERVER_TMPDIR", "PMIX_SYSTEM_TMPDIR",
                                                         "PMIX_SERVER_NSPACE", "PMIX_SERVER_RANK",
                                                         "PMIX_GDS_MODULE", "PMIX_EVENT_BASE",
                                                         "PMIX_HOSTNAME", "PMIX_NODEID", "PMIX_TCP_IF_INCLUDE",
                                                         "PMIX_TCP_IPV4_PORT", "PMIX_TCP_IPV6_PORT",
                                                         "PMIX_TCP_DISABLE_IPV4", "PMIX_TCP_DISABLE_IPV6",
                                                         "PMIX_SERVER_REMOTE_CONNECTIONS", "PMIX_TCP_REPORT_URI",
                                                         "PMIX_SERVER_SESSION_SUPPORT", "PMIX_SERVER_SYSTEM_SUPPORT",
                                                         "PMIX_SERVER_TOOL_SUPPORT", "PMIX_LAUNCHER_RENDEZVOUS_FILE",
                                                         "PMIX_SERVER_ENABLE_MONITORING", NULL}},
    {.function = "PMIx_server_finalize", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_generate_regex", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_generate_ppn", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_register_nspace", .attrs = (char *[]){"PMIX_REGISTER_NODATA", NULL}},
    {.function = "PMIx_server_deregister_nspace", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_register_client", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_deregister_client", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_setup_fork", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_dmodex_request", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_setup_application", .attrs = (char *[]){"PMIX_SETUP_APP_ENVARS", "PMIX_SETUP_APP_ALL",
                                                                      "PMIX_SETUP_APP_NONENVARS", "PMIX_ALLOC_FABRIC",
                                                                      "PMIX_ALLOC_FABRIC_SEC_KEY", "PMIX_ALLOC_FABRIC_ID",
                                                                      "PMIX_ALLOC_FABRIC_TYPE", "PMIX_ALLOC_FABRIC_PLANE",
                                                                      "PMIX_ALLOC_FABRIC_ENDPTS", NULL}},
    {.function = "PMIx_server_setup_local_support", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_server_IOF_deliver", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_server_collect_inventory", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_server_deliver_inventory", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_Register_attributes", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_register_fabric", .attrs = (char *[]){"PMIX_FABRIC_PLANE", "PMIX_FABRIC_IDENTIFIER",
                                                                    "PMIX_FABRIC_VENDOR", NULL}},
    {.function = "PMIx_server_update_fabric", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_deregister_fabric", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_get_vertex_info", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_server_get_index", .attrs = (char *[]){"PMIX_HOSTNAME", "PMIX_NODEID", "PMIX_FABRIC_DEVICE_NAME",
                                                              "PMIX_FABRIC_DEVICE_VENDOR", "PMIX_FABRIC_DEVICE_BUS_TYPE",
                                                              "PMIX_FABRIC_DEVICE_PCI_DEVID", NULL}},
    {.function = ""}
};

/*****    REGISTER SERVER ATTRS    *****/
static bool server_attrs_regd = false;

PMIX_EXPORT pmix_status_t pmix_register_server_attrs(void)
{
    pmix_status_t rc = PMIX_SUCCESS;
    size_t n;

    if (server_attrs_regd) {
        return PMIX_SUCCESS;
    }
    server_attrs_regd = true;

    for (n=0; 0 != strlen(server_fns[n].function); n++) {
        rc = process_reg(PMIX_SERVER_ATTRIBUTES,
                         server_fns[n].function, server_fns[n].attrs);
        if (PMIX_SUCCESS != rc) {
            break;
        }
    }

    return rc;
}

static pmix_attr_init_t tool_fns[] = {
    {.function = "PMIx_tool_init", .attrs = (char *[]){"PMIX_GDS_MODULE", "PMIX_EVENT_BASE", "PMIX_HOSTNAME",
                                                       "PMIX_NODEID", "PMIX_TOOL_DO_NOT_CONNECT", "PMIX_TOOL_NSPACE",
                                                       "PMIX_TOOL_RANK", "PMIX_FWD_STDIN", "PMIX_LAUNCHER",
                                                       "PMIX_SERVER_TMPDIR", "PMIX_SYSTEM_TMPDIR",
                                                       "PMIX_TOOL_CONNECT_OPTIONAL", "PMIX_RECONNECT_SERVER",
                                                       "PMIX_TOOL_ATTACHMENT_FILE", "PMIX_CONNECT_MAX_RETRIES",
                                                       "PMIX_CONNECT_RETRY_DELAY", "PMIX_CONNECT_TO_SYSTEM",
                                                       "PMIX_CONNECT_SYSTEM_FIRST", "PMIX_SERVER_PIDINFO",
                                                       "PMIX_SERVER_NSPACE", "PMIX_SERVER_URI", NULL}},
    {.function = "PMIx_tool_finalize", .attrs = (char *[]){"N/A", NULL}},
    {.function = "PMIx_tool_connect_to_server", .attrs = (char *[]){"PMIX_CONNECT_TO_SYSTEM", "PMIX_CONNECT_SYSTEM_FIRST",
                                                                    "PMIX_SERVER_PIDINFO", "PMIX_SERVER_NSPACE",
                                                                    "PMIX_SERVER_URI", "PMIX_CONNECT_RETRY_DELAY",
                                                                    "PMIX_CONNECT_MAX_RETRIES", "PMIX_RECONNECT_SERVER",
                                                                    "PMIX_TOOL_ATTACHMENT_FILE", NULL}},
    {.function = "PMIx_IOF_pull", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_IOF_deregister", .attrs = (char *[]){"NONE", NULL}},
    {.function = "PMIx_IOF_push", .attrs = (char *[]){"PMIX_IOF_PUSH_STDIN", "PMIX_IOF_COMPLETE", NULL}},
    {.function = ""}
};

/*****    REGISTER TOOL ATTRS    *****/
static bool tool_attrs_regd = false;

PMIX_EXPORT pmix_status_t pmix_register_tool_attrs(void)
{
    pmix_status_t rc = PMIX_SUCCESS;
    size_t n;

    if (tool_attrs_regd) {
        return PMIX_SUCCESS;
    }
    tool_attrs_regd = true;

    for (n=0; 0 != strlen(tool_fns[n].function); n++) {
        rc = process_reg(PMIX_TOOL_ATTRIBUTES,
                         tool_fns[n].function, tool_fns[n].attrs);
        if (PMIX_SUCCESS != rc) {
            break;
        }
    }

    return rc;
}

/*****   PROCESS QUERY ATTRS    *****/
static void _get_attrs(pmix_list_t *lst,
                       pmix_info_t *info,
                       pmix_list_t *attrs)
{
    pmix_attribute_trk_t *trk, *tptr;
    pmix_infolist_t *ip;
    pmix_data_array_t *darray;
    pmix_regattr_t *regarray;
    size_t m, nattr;
    char **fns;
    const pmix_regattr_input_t *dptr;

    /* the value in the info is a comma-delimited list of
     * functions whose attributes are being requested */
    fns = pmix_argv_split(info->value.data.string, ',');

    /* search the list for these functions */
    PMIX_LIST_FOREACH(tptr, attrs, pmix_attribute_trk_t) {
        trk = NULL;
        for (m=0; NULL != fns[m] && NULL == trk; m++) {
            if (0 == strcmp(fns[m], tptr->function) ||
                0 == strcmp(fns[m], "all")) {
                trk = tptr;
                break;
            }
        }
        if (NULL == trk || NULL == trk->attrs) {
            /* function wasn't found - no attrs
             * registered for it */
            continue;
        }
        /* add the found attrs to the results */
        ip = PMIX_NEW(pmix_infolist_t);
        PMIX_LOAD_KEY(ip->info.key, tptr->function);
        /* create the data array to hold the results */
        nattr = pmix_argv_count(tptr->attrs);
        PMIX_DATA_ARRAY_CREATE(darray, nattr, PMIX_REGATTR);
        ip->info.value.type = PMIX_DATA_ARRAY;
        ip->info.value.data.darray = darray;
        regarray = (pmix_regattr_t*)darray->array;
        for (m=0; m < nattr; m++) {
            regarray[m].name = strdup(tptr->attrs[m]);
            PMIX_LOAD_KEY(regarray[m].string, pmix_attributes_lookup(tptr->attrs[m]));
            dptr = pmix_attributes_lookup_term(tptr->attrs[m]);
            if (NULL == dptr) {
                PMIX_RELEASE(ip);
                return;
            }
            regarray[m].type = dptr->type;
            regarray[m].description = pmix_argv_copy(dptr->description);
        }
        pmix_list_append(lst, &ip->super);
    }
    pmix_argv_free(fns);
}

static void _get_fns(pmix_list_t *lst,
                     pmix_info_t *info,
                     pmix_list_t *attrs)
{
    pmix_attribute_trk_t *tptr;
    pmix_infolist_t *ip;
    char **fns = NULL, *tmp;

    /* search the list for these functions */
    PMIX_LIST_FOREACH(tptr, attrs, pmix_attribute_trk_t) {
        pmix_argv_append_nosize(&fns, tptr->function);
    }
    if (0 < pmix_argv_count(fns)) {
        ip = PMIX_NEW(pmix_infolist_t);
        tmp = pmix_argv_join(fns, ',');
        PMIX_INFO_LOAD(&ip->info, info->key, tmp, PMIX_STRING);
        pmix_list_append(lst, &ip->super);
        pmix_argv_free(fns);
    }
}

static void _local_relcb(void *cbdata)
{
    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
    PMIX_RELEASE(cd);
}

static void relcbfunc(void *cbdata)
{
    pmix_shift_caddy_t *cd = (pmix_shift_caddy_t*)cbdata;

    pmix_output_verbose(2, pmix_globals.debug_output,
                        "pmix:query release callback");

    if (NULL != cd->info) {
        PMIX_INFO_FREE(cd->info, cd->ninfo);
    }
    PMIX_RELEASE(cd);
}
static void query_cbfunc(struct pmix_peer_t *peer,
                         pmix_ptl_hdr_t *hdr,
                         pmix_buffer_t *buf, void *cbdata)
{
    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
    pmix_status_t rc;
    pmix_shift_caddy_t *results;
    int cnt;

    pmix_output_verbose(2, pmix_globals.debug_output,
                        "pmix:attrs:query cback from server");

    results = PMIX_NEW(pmix_shift_caddy_t);

    /* unpack the status */
    cnt = 1;
    PMIX_BFROPS_UNPACK(rc, peer, buf, &results->status, &cnt, PMIX_STATUS);
    if (PMIX_SUCCESS != rc) {
        results->status = rc;
        goto complete;
    }
    if (PMIX_SUCCESS != results->status) {
        goto complete;
    }

    /* unpack any returned data */
    cnt = 1;
    PMIX_BFROPS_UNPACK(rc, peer, buf, &results->ninfo, &cnt, PMIX_SIZE);
    if (PMIX_SUCCESS != rc) {
        results->status = rc;
        goto complete;
    }
    if (0 < results->ninfo) {
        PMIX_INFO_CREATE(results->info, results->ninfo);
        cnt = results->ninfo;
        PMIX_BFROPS_UNPACK(rc, peer, buf, results->info, &cnt, PMIX_INFO);
        if (PMIX_SUCCESS != rc) {
            results->status = rc;
            goto complete;
        }
    }

  complete:
    pmix_output_verbose(2, pmix_globals.debug_output,
                        "pmix:query cback from server releasing");
    /* release the caller */
    if (NULL != cd->cbfunc) {
        cd->cbfunc(results->status, results->info, results->ninfo, cd->cbdata, relcbfunc, results);
    }
    PMIX_RELEASE(cd);
}

PMIX_EXPORT void pmix_attrs_query_support(int sd, short args, void *cbdata)
{
    pmix_query_caddy_t *cd = (pmix_query_caddy_t*)cbdata;
    pmix_infolist_t *info, *head;
    pmix_list_t kyresults;
    size_t n, m, p;
    pmix_info_t *iptr;
    pmix_data_array_t *darray;
    pmix_buffer_t *msg;
    pmix_cmd_t cmd = PMIX_QUERY_CMD;
    pmix_status_t rc;

    PMIX_ACQUIRE_THREAD(&pmix_global_lock);

    for (n=0; n < cd->nqueries; n++) {
        if (0 != strcmp(cd->queries[n].keys[0], PMIX_QUERY_ATTRIBUTE_SUPPORT)) {
            /* skip this one */
            continue;
        }
        head = NULL;
        for (m=0; m < cd->queries[n].nqual; m++) {
            PMIX_CONSTRUCT(&kyresults, pmix_list_t);
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_CLIENT_ATTRIBUTES)) {
                /* everyone has access to the client attrs */
                _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &client_attrs);
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_CLIENT_FUNCTIONS)) {
                /* everyone has access to the client functions */
                _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &client_attrs);
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_SERVER_ATTRIBUTES)) {
                /* if I am a server, add in my attrs */
                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &server_attrs);
                 } else {
                    /* we need to ask our server for them */
                    PMIX_LIST_DESTRUCT(&kyresults);
                    goto query;
                }
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_SERVER_FUNCTIONS)) {
                /* if I am a server, add in my fns */
                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &server_attrs);
                } else {
                    /* we need to ask our server for them */
                    PMIX_LIST_DESTRUCT(&kyresults);
                    goto query;
                }
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_TOOL_ATTRIBUTES)) {
                if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &tool_attrs);
                }
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_TOOL_FUNCTIONS)) {
                if (PMIX_PEER_IS_TOOL(pmix_globals.mypeer)) {
                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &tool_attrs);
                }
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_HOST_ATTRIBUTES)) {
                /* if I am a server, add in the host's */
                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
                    _get_attrs(&kyresults, &cd->queries[n].qualifiers[m], &host_attrs);
                } else {
                    /* we need to ask our server for them */
                    PMIX_LIST_DESTRUCT(&kyresults);
                    goto query;
                }
            }
            if (NULL == cd->queries[n].qualifiers ||
                PMIX_CHECK_KEY(&cd->queries[n].qualifiers[m], PMIX_HOST_FUNCTIONS)) {
                /* if I am a server, add in the host's */
                if (PMIX_PEER_IS_SERVER(pmix_globals.mypeer)) {
                    _get_fns(&kyresults, &cd->queries[n].qualifiers[m], &host_attrs);
                } else {
                    /* we need to ask our server for them */
                    PMIX_LIST_DESTRUCT(&kyresults);
                    goto query;
                }
            }
            if (0 < (p = pmix_list_get_size(&kyresults))) {
                head = PMIX_NEW(pmix_infolist_t);
                PMIX_LOAD_KEY(head->info.key, cd->queries[n].keys[m]);
                head->info.value.type = PMIX_DATA_ARRAY;
                /* create the data array to hold the results */
                PMIX_DATA_ARRAY_CREATE(darray, p, PMIX_INFO);
                head->info.value.data.darray = darray;
                iptr = (pmix_info_t*)darray->array;
                p = 0;
                PMIX_LIST_FOREACH(info, &kyresults, pmix_infolist_t) {
                    PMIX_INFO_XFER(&iptr[p], &info->info);
                    ++p;
                }
                pmix_list_append(&cd->results, &head->super);
            }
            PMIX_LIST_DESTRUCT(&kyresults);
        }
    }
    /* prep the response by converting the list
     * of results into an array */
    if (0 < (cd->ninfo = pmix_list_get_size(&cd->results))) {
        PMIX_INFO_CREATE(cd->info, cd->ninfo);
        n = 0;
        PMIX_LIST_FOREACH(info, &cd->results, pmix_infolist_t) {
            PMIX_INFO_XFER(&cd->info[n], &info->info);
            ++n;
        }
        cd->status = PMIX_SUCCESS;
    } else {
        cd->status = PMIX_ERR_NOT_FOUND;
    }
    PMIX_RELEASE_THREAD(&pmix_global_lock);
    goto release;

  query:
    /* if we aren't connected, don't attempt to send */
    if (!pmix_globals.connected) {
        PMIX_RELEASE_THREAD(&pmix_global_lock);
        cd->status = PMIX_ERR_NOT_FOUND;
        goto release;
    }
    PMIX_RELEASE_THREAD(&pmix_global_lock);

    /* relay this request to the server */
    msg = PMIX_NEW(pmix_buffer_t);
    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                     msg, &cmd, 1, PMIX_COMMAND);
    if (PMIX_SUCCESS != rc) {
        PMIX_RELEASE(msg);
        cd->status = rc;
        goto release;
    }
    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                     msg, &cd->nqueries, 1, PMIX_SIZE);
    if (PMIX_SUCCESS != rc) {
        PMIX_RELEASE(msg);
        cd->status = rc;
        goto release;
    }
    PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver,
                     msg, cd->queries, cd->nqueries, PMIX_QUERY);
    if (PMIX_SUCCESS != rc) {
        PMIX_RELEASE(msg);
        cd->status = rc;
        goto release;
    }

    pmix_output_verbose(2, pmix_globals.debug_output,
                        "pmix:query sending to server");
    PMIX_PTL_SEND_RECV(rc, pmix_client_globals.myserver,
                       msg, query_cbfunc, (void*)cd);
    if (PMIX_SUCCESS != rc) {
        cd->status = rc;
        goto release;
    }
    return;

  release:
    if (NULL != cd->cbfunc) {
        cd->cbfunc(cd->status, cd->info, cd->ninfo, cd, _local_relcb, cd);
        return;
    }

    PMIX_RELEASE(cd);
}

/*****   LOCATE A GIVEN ATTRIBUTE    *****/
PMIX_EXPORT const char* pmix_attributes_lookup(char *attr)
{
    size_t n;

    for (n=0; 0 != strlen(dictionary[n].name); n++) {
        if (0 == strcasecmp(dictionary[n].name, attr)) {
            return dictionary[n].string;
        }
    }
    return NULL;
}

PMIX_EXPORT const char* pmix_attributes_reverse_lookup(char *attrstring)
{
    size_t n;

    for (n=0; 0 != strlen(dictionary[n].name); n++) {
        if (0 == strcasecmp(dictionary[n].string, attrstring)) {
            return dictionary[n].name;
        }
    }
    return NULL;
}

PMIX_EXPORT const pmix_regattr_input_t* pmix_attributes_lookup_term(char *attr)
{
    size_t n;

    for (n=0; 0 != strlen(dictionary[n].name); n++) {
        if (0 == strcmp(dictionary[n].name, attr)) {
            return &dictionary[n];
        }
    }
    return NULL;
}

/*****   PRINT QUERY FUNCTIONS RESULTS   *****/
PMIX_EXPORT char** pmix_attributes_print_functions(char *level)
{
    char *title1 = "CLIENT SUPPORTED FUNCTIONS: ";
    char *title2 = "SERVER SUPPORTED FUNCTIONS: ";
    char *title3 = "HOST SUPPORTED FUNCTIONS: ";
    char *title4 = "TOOL SUPPORTED FUNCTIONS: ";
    char **ans = NULL;
    pmix_list_t *lst;
    pmix_attribute_trk_t *fnptr;

    /* select title */
    if (0 == strcmp(level, PMIX_CLIENT_FUNCTIONS)) {
        pmix_argv_append_nosize(&ans, title1);
        lst = &client_attrs;
    } else if (0 == strcmp(level, PMIX_SERVER_FUNCTIONS)) {
        pmix_argv_append_nosize(&ans, title2);
        lst = &server_attrs;
    } else if (0 == strcmp(level, PMIX_HOST_FUNCTIONS)) {
        pmix_argv_append_nosize(&ans, title3);
        lst = &host_attrs;
    } else if (0 == strcmp(level, PMIX_TOOL_FUNCTIONS)) {
        pmix_argv_append_nosize(&ans, title4);
        lst = &tool_attrs;
    } else {
        return NULL;
    }

    PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
        pmix_argv_append_nosize(&ans, fnptr->function);
    }
    return ans;
}

/*****   PRINT QUERY ATTRS RESULTS   *****/

#define PMIX_PRINT_NAME_COLUMN_WIDTH      35
#define PMIX_PRINT_STRING_COLUMN_WIDTH    25
#define PMIX_PRINT_TYPE_COLUMN_WIDTH      20
#define PMIX_PRINT_ATTR_COLUMN_WIDTH     141

void pmix_attributes_print_attrs(char ***ans, char *function,
                                 pmix_regattr_t *attrs,
                                 size_t nattrs)
{
    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH], *tmp;
    size_t n, m, len;

    /* print the function */
    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
    m = 0;
    for (n=0; n < strlen(function); n++) {
        line[m] = function[n];
        ++m;
    }
    line[m++] = ':';
    line[m] = '\0';
    pmix_argv_append_nosize(ans, line);

    for (n=0; n < nattrs; n++) {
        memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
        line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
        len = strlen(attrs[n].name);
        if (PMIX_PRINT_NAME_COLUMN_WIDTH < len) {
            len = PMIX_PRINT_NAME_COLUMN_WIDTH;
        }
        memcpy(line, attrs[n].name, len);

        len = strlen(attrs[n].string);
        if (PMIX_PRINT_STRING_COLUMN_WIDTH < len) {
            len = PMIX_PRINT_STRING_COLUMN_WIDTH;
        }
        memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+2], attrs[n].string, len);

        tmp = (char*)PMIx_Data_type_string(attrs[n].type);
        len = strlen(tmp);
        if (PMIX_PRINT_STRING_COLUMN_WIDTH < len) {
            len = PMIX_PRINT_STRING_COLUMN_WIDTH;
        }
        memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+PMIX_PRINT_STRING_COLUMN_WIDTH+4], tmp, len);

        for (m=0; NULL != attrs[n].description[m]; m++) {
            len = strlen(attrs[n].description[m]);
            memcpy(&line[PMIX_PRINT_NAME_COLUMN_WIDTH+PMIX_PRINT_STRING_COLUMN_WIDTH+PMIX_PRINT_TYPE_COLUMN_WIDTH+6], attrs[n].description[m], len);
            line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';  // ensure NULL termination
            pmix_argv_append_nosize(ans, line);
            memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
            line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
        }
    }
}

void pmix_attributes_print_headers(char ***ans, char *level)
{
    size_t n, m, left;
    char *title1 = "CLIENT SUPPORTED ATTRIBUTES: ";
    char *title2 = "SERVER SUPPORTED ATTRIBUTES: ";
    char *title3 = "HOST SUPPORTED ATTRIBUTES: ";
    char *title4 = "TOOL SUPPORTED ATTRIBUTES: ";
    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH];

    /* select title */
    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
        pmix_argv_append_nosize(ans, title1);
    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
        pmix_argv_append_nosize(ans, title2);
    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
        pmix_argv_append_nosize(ans, title3);
    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
        pmix_argv_append_nosize(ans, title4);
    } else {
        return;
    }

    /* print the column headers */
    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
    line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
    left = PMIX_PRINT_NAME_COLUMN_WIDTH/2 - 1;
    memcpy(&line[left], "NAME", 4);

    left = 3 + PMIX_PRINT_NAME_COLUMN_WIDTH + (PMIX_PRINT_STRING_COLUMN_WIDTH/2) - 2;
    memcpy(&line[left], "STRING", 6);

    left = 3 + PMIX_PRINT_NAME_COLUMN_WIDTH + PMIX_PRINT_STRING_COLUMN_WIDTH + (PMIX_PRINT_TYPE_COLUMN_WIDTH/2) - 2;
    memcpy(&line[left], "TYPE", 4);

    left = PMIX_PRINT_NAME_COLUMN_WIDTH + PMIX_PRINT_STRING_COLUMN_WIDTH + PMIX_PRINT_TYPE_COLUMN_WIDTH +
           ((PMIX_PRINT_ATTR_COLUMN_WIDTH-PMIX_PRINT_NAME_COLUMN_WIDTH-PMIX_PRINT_STRING_COLUMN_WIDTH-PMIX_PRINT_TYPE_COLUMN_WIDTH)/2) - 3 - strlen("DESCRIPTION")/2;
    memcpy(&line[left], "DESCRIPTION", strlen("DESCRIPTION"));
    left += strlen("DESCRIPTION") + 1;
    line[left] = '\0';
    pmix_argv_append_nosize(ans, line);

    /* print the dashes under the column headers */
    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
    line[PMIX_PRINT_ATTR_COLUMN_WIDTH-1] = '\0';
    m=0;
    for (n=0; n < PMIX_PRINT_NAME_COLUMN_WIDTH; n++) {
        line[m] = '-';
        ++m;
    }
    m += 2; // leave gap
    for (n=0; n < PMIX_PRINT_STRING_COLUMN_WIDTH; n++) {
        line[m] = '-';
        ++m;
    }
    m += 2; // leave gap
    for (n=0; n < PMIX_PRINT_TYPE_COLUMN_WIDTH; n++) {
        line[m] = '-';
        ++m;
    }
    m += 2; // leave gap
    while (m < PMIX_PRINT_ATTR_COLUMN_WIDTH-1) {
        line[m] = '-';
        ++m;
    }
    pmix_argv_append_nosize(ans, line);
}


PMIX_EXPORT char** pmix_attributes_print_attr(char *level, char *function)
{
    size_t n, m, nattr;
    char **tmp, **ans=NULL;
    pmix_list_t *lst;
    pmix_attribute_trk_t *fnptr;
    char line[PMIX_PRINT_ATTR_COLUMN_WIDTH];
    pmix_regattr_t *rptr;
    const pmix_regattr_input_t *dptr;

    /* select title */
    if (0 == strcmp(level, PMIX_CLIENT_ATTRIBUTES)) {
        lst = &client_attrs;
    } else if (0 == strcmp(level, PMIX_SERVER_ATTRIBUTES)) {
        lst = &server_attrs;
    } else if (0 == strcmp(level, PMIX_HOST_ATTRIBUTES)) {
        lst = &host_attrs;
    } else if (0 == strcmp(level, PMIX_TOOL_ATTRIBUTES)) {
        lst = &tool_attrs;
    } else {
        return NULL;
    }

    /* print the column headers */
    pmix_attributes_print_headers(&ans, level);
    memset(line, ' ', PMIX_PRINT_ATTR_COLUMN_WIDTH);
    line[1] = '\0';

    /* can be comma-delimited list of functions */
    tmp = pmix_argv_split(function, ',');
    for (n=0; NULL != tmp[n]; n++) {
        PMIX_LIST_FOREACH(fnptr, lst, pmix_attribute_trk_t) {
            if (0 == strcmp(tmp[n], "all") ||
                0 == strcmp(tmp[n], fnptr->function)) {
                /* create an array of pmix_regattr_t for this function's attributes */
                nattr = pmix_argv_count(fnptr->attrs);
                PMIX_REGATTR_CREATE(rptr, nattr);
                for (m=0; m < nattr; m++) {
                    rptr[m].name = strdup(fnptr->attrs[m]);
                    PMIX_LOAD_KEY(rptr[m].string, pmix_attributes_lookup(fnptr->attrs[m]));
                    dptr = pmix_attributes_lookup_term(fnptr->attrs[m]);
                    if (NULL == dptr) {
                        pmix_argv_free(tmp);
                        pmix_argv_free(ans);
                        PMIX_REGATTR_FREE(rptr, nattr);
                        return NULL;
                    }
                    rptr[m].type = dptr->type;
                    rptr[m].description = pmix_argv_copy(dptr->description);
                }
                pmix_attributes_print_attrs(&ans, fnptr->function, rptr, nattr);
                PMIX_REGATTR_FREE(rptr, nattr);
                pmix_argv_append_nosize(&ans, line);
                if (0 == strcmp(tmp[n], fnptr->function)) {
                    break;
                }
            }
        }
    }
    pmix_argv_free(tmp);


    return ans;
}
