lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
lrmd_tls_connected
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd_tls_send_msg
lrmd_tls_recv_reply
lrmd_tls_send
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
remote_proxy_check
lrmd_handshake
lrmd_ipc_connect
set_key
lrmd_tls_set_key
lrmd_gnutls_global_init
report_async_connection_result
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
   1 /*
   2  * Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  *
  18  */
  19 
  20 #include <crm_internal.h>
  21 
  22 #include <unistd.h>
  23 #include <stdlib.h>
  24 #include <stdio.h>
  25 #include <stdarg.h>
  26 #include <string.h>
  27 #include <ctype.h>
  28 
  29 #include <sys/types.h>
  30 #include <sys/wait.h>
  31 
  32 #include <glib.h>
  33 #include <dirent.h>
  34 
  35 #include <crm/crm.h>
  36 #include <crm/lrmd.h>
  37 #include <crm/services.h>
  38 #include <crm/common/mainloop.h>
  39 #include <crm/common/ipcs.h>
  40 #include <crm/msg_xml.h>
  41 
  42 #include <crm/stonith-ng.h>
  43 
  44 #ifdef HAVE_GNUTLS_GNUTLS_H
  45 #  undef KEYFILE
  46 #  include <gnutls/gnutls.h>
  47 #endif
  48 
  49 #include <sys/socket.h>
  50 #include <netinet/in.h>
  51 #include <netinet/ip.h>
  52 #include <arpa/inet.h>
  53 #include <netdb.h>
  54 
  55 #define MAX_TLS_RECV_WAIT 10000
  56 
  57 CRM_TRACE_INIT_DATA(lrmd);
  58 
  59 static int lrmd_api_disconnect(lrmd_t * lrmd);
  60 static int lrmd_api_is_connected(lrmd_t * lrmd);
  61 
  62 /* IPC proxy functions */
  63 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  64 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  65 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  66 
  67 #ifdef HAVE_GNUTLS_GNUTLS_H
  68 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  69 gnutls_psk_client_credentials_t psk_cred_s;
  70 int lrmd_tls_set_key(gnutls_datum_t * key);
  71 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  72 static int global_remote_msg_id = 0;
  73 int lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type);
  74 static void lrmd_tls_connection_destroy(gpointer userdata);
  75 #endif
  76 
  77 typedef struct lrmd_private_s {
  78     enum client_type type;
  79     char *token;
  80     mainloop_io_t *source;
  81 
  82     /* IPC parameters */
  83     crm_ipc_t *ipc;
  84 
  85     crm_remote_t *remote;
  86 
  87     /* Extra TLS parameters */
  88     char *remote_nodename;
  89 #ifdef HAVE_GNUTLS_GNUTLS_H
  90     char *server;
  91     int port;
  92     gnutls_psk_client_credentials_t psk_cred_c;
  93 
  94     /* while the async connection is occurring, this is the id
  95      * of the connection timeout timer. */
  96     int async_timer;
  97     int sock;
  98     /* since tls requires a round trip across the network for a
  99      * request/reply, there are times where we just want to be able
 100      * to send a request from the client and not wait around (or even care
 101      * about) what the reply is. */
 102     int expected_late_replies;
 103     GList *pending_notify;
 104     crm_trigger_t *process_notify;
 105 #endif
 106 
 107     lrmd_event_callback callback;
 108 
 109     /* Internal IPC proxy msg passing for remote guests */
 110     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 111     void *proxy_callback_userdata;
 112     char *peer_version;
 113 } lrmd_private_t;
 114 
 115 static lrmd_list_t *
 116 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 117 {
 118     lrmd_list_t *p, *end;
 119 
 120     p = calloc(1, sizeof(lrmd_list_t));
 121     p->val = strdup(value);
 122 
 123     end = head;
 124     while (end && end->next) {
 125         end = end->next;
 126     }
 127 
 128     if (end) {
 129         end->next = p;
 130     } else {
 131         head = p;
 132     }
 133 
 134     return head;
 135 }
 136 
 137 void
 138 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 139 {
 140     lrmd_list_t *p;
 141 
 142     while (head) {
 143         char *val = (char *)head->val;
 144 
 145         p = head->next;
 146         free(val);
 147         free(head);
 148         head = p;
 149     }
 150 }
 151 
 152 lrmd_key_value_t *
 153 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 154 {
 155     lrmd_key_value_t *p, *end;
 156 
 157     p = calloc(1, sizeof(lrmd_key_value_t));
 158     p->key = strdup(key);
 159     p->value = strdup(value);
 160 
 161     end = head;
 162     while (end && end->next) {
 163         end = end->next;
 164     }
 165 
 166     if (end) {
 167         end->next = p;
 168     } else {
 169         head = p;
 170     }
 171 
 172     return head;
 173 }
 174 
 175 void
 176 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 177 {
 178     lrmd_key_value_t *p;
 179 
 180     while (head) {
 181         p = head->next;
 182         free(head->key);
 183         free(head->value);
 184         free(head);
 185         head = p;
 186     }
 187 }
 188 
 189 lrmd_event_data_t *
 190 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 191 {
 192     lrmd_event_data_t *copy = NULL;
 193 
 194     copy = calloc(1, sizeof(lrmd_event_data_t));
 195 
 196     /* This will get all the int values.
 197      * we just have to be careful not to leave any
 198      * dangling pointers to strings. */
 199     memcpy(copy, event, sizeof(lrmd_event_data_t));
 200 
 201     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
 202     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
 203     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
 204     copy->output = event->output ? strdup(event->output) : NULL;
 205     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
 206     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
 207     copy->params = crm_str_table_dup(event->params);
 208 
 209     return copy;
 210 }
 211 
 212 void
 213 lrmd_free_event(lrmd_event_data_t * event)
     /*  */
 214 {
 215     if (!event) {
 216         return;
 217     }
 218 
 219     /* free gives me grief if i try to cast */
 220     free((char *)event->rsc_id);
 221     free((char *)event->op_type);
 222     free((char *)event->user_data);
 223     free((char *)event->output);
 224     free((char *)event->exit_reason);
 225     free((char *)event->remote_nodename);
 226     if (event->params) {
 227         g_hash_table_destroy(event->params);
 228     }
 229     free(event);
 230 }
 231 
 232 static int
 233 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 234 {
 235     const char *type;
 236     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 237     lrmd_private_t *native = lrmd->private;
 238     lrmd_event_data_t event = { 0, };
 239 
 240     if (proxy_session != NULL) {
 241         /* this is proxy business */
 242         lrmd_internal_proxy_dispatch(lrmd, msg);
 243         return 1;
 244     } else if (!native->callback) {
 245         /* no callback set */
 246         crm_trace("notify event received but client has not set callback");
 247         return 1;
 248     }
 249 
 250     event.remote_nodename = native->remote_nodename;
 251     type = crm_element_value(msg, F_LRMD_OPERATION);
 252     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 253     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 254 
 255     if (crm_str_eq(type, LRMD_OP_RSC_REG, TRUE)) {
 256         event.type = lrmd_event_register;
 257     } else if (crm_str_eq(type, LRMD_OP_RSC_UNREG, TRUE)) {
 258         event.type = lrmd_event_unregister;
 259     } else if (crm_str_eq(type, LRMD_OP_RSC_EXEC, TRUE)) {
 260         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 261         crm_element_value_int(msg, F_LRMD_RSC_INTERVAL, &event.interval);
 262         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 263         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 264         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 265         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 266 
 267         crm_element_value_int(msg, F_LRMD_RSC_RUN_TIME, (int *)&event.t_run);
 268         crm_element_value_int(msg, F_LRMD_RSC_RCCHANGE_TIME, (int *)&event.t_rcchange);
 269         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 270         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 271 
 272         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 273         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 274         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
 275         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
 276         event.type = lrmd_event_exec_complete;
 277 
 278         event.params = xml2list(msg);
 279     } else if (crm_str_eq(type, LRMD_OP_NEW_CLIENT, TRUE)) {
 280         event.type = lrmd_event_new_client;
 281     } else if (crm_str_eq(type, LRMD_OP_POKE, TRUE)) {
 282         event.type = lrmd_event_poke;
 283     } else {
 284         return 1;
 285     }
 286 
 287     crm_trace("op %s notify event received", type);
 288     native->callback(&event);
 289 
 290     if (event.params) {
 291         g_hash_table_destroy(event.params);
 292     }
 293     return 1;
 294 }
 295 
 296 static int
 297 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 298 {
 299     lrmd_t *lrmd = userdata;
 300     lrmd_private_t *native = lrmd->private;
 301     xmlNode *msg;
 302     int rc;
 303 
 304     if (!native->callback) {
 305         /* no callback set */
 306         return 1;
 307     }
 308 
 309     msg = string2xml(buffer);
 310     rc = lrmd_dispatch_internal(lrmd, msg);
 311     free_xml(msg);
 312     return rc;
 313 }
 314 
 315 #ifdef HAVE_GNUTLS_GNUTLS_H
 316 static void
 317 lrmd_free_xml(gpointer userdata)
     /*  */
 318 {
 319     free_xml((xmlNode *) userdata);
 320 }
 321 
 322 static int
 323 lrmd_tls_connected(lrmd_t * lrmd)
     /*  */
 324 {
 325     lrmd_private_t *native = lrmd->private;
 326 
 327     if (native->remote->tls_session) {
 328         return TRUE;
 329     }
 330 
 331     return FALSE;
 332 }
 333 
 334 static int
 335 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 336 {
 337     lrmd_t *lrmd = userdata;
 338     lrmd_private_t *native = lrmd->private;
 339     xmlNode *xml = NULL;
 340     int rc = 0;
 341     int disconnected = 0;
 342 
 343     if (lrmd_tls_connected(lrmd) == FALSE) {
 344         crm_trace("tls dispatch triggered after disconnect");
 345         return 0;
 346     }
 347 
 348     crm_trace("tls_dispatch triggered");
 349 
 350     /* First check if there are any pending notifies to process that came
 351      * while we were waiting for replies earlier. */
 352     if (native->pending_notify) {
 353         GList *iter = NULL;
 354 
 355         crm_trace("Processing pending notifies");
 356         for (iter = native->pending_notify; iter; iter = iter->next) {
 357             lrmd_dispatch_internal(lrmd, iter->data);
 358         }
 359         g_list_free_full(native->pending_notify, lrmd_free_xml);
 360         native->pending_notify = NULL;
 361     }
 362 
 363     /* Next read the current buffer and see if there are any messages to handle. */
 364     rc = crm_remote_ready(native->remote, 0);
 365     if (rc == 0) {
 366         /* nothing to read, see if any full messages are already in buffer. */
 367         xml = crm_remote_parse_buffer(native->remote);
 368     } else if (rc < 0) {
 369         disconnected = 1;
 370     } else {
 371         crm_remote_recv(native->remote, -1, &disconnected);
 372         xml = crm_remote_parse_buffer(native->remote);
 373     }
 374     while (xml) {
 375         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 376         if (safe_str_eq(msg_type, "notify")) {
 377             lrmd_dispatch_internal(lrmd, xml);
 378         } else if (safe_str_eq(msg_type, "reply")) {
 379             if (native->expected_late_replies > 0) {
 380                 native->expected_late_replies--;
 381             } else {
 382                 int reply_id = 0;
 383                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 384                 /* if this happens, we want to know about it */
 385                 crm_err("Got outdated reply %d", reply_id);
 386             }
 387         }
 388         free_xml(xml);
 389         xml = crm_remote_parse_buffer(native->remote);
 390     }
 391 
 392     if (disconnected) {
 393         crm_info("Server disconnected while reading remote server msg.");
 394         lrmd_tls_disconnect(lrmd);
 395         return 0;
 396     }
 397     return 1;
 398 }
 399 #endif
 400 
 401 /* Not used with mainloop */
 402 int
 403 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 404 {
 405     lrmd_private_t *native = lrmd->private;
 406 
 407     switch (native->type) {
 408         case CRM_CLIENT_IPC:
 409             return crm_ipc_ready(native->ipc);
 410 
 411 #ifdef HAVE_GNUTLS_GNUTLS_H
 412         case CRM_CLIENT_TLS:
 413             if (native->pending_notify) {
 414                 return 1;
 415             }
 416 
 417             return crm_remote_ready(native->remote, 0);
 418 #endif
 419         default:
 420             crm_err("Unsupported connection type: %d", native->type);
 421     }
 422 
 423     return 0;
 424 }
 425 
 426 /* Not used with mainloop */
 427 bool
 428 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 429 {
 430     lrmd_private_t *private = NULL;
 431 
 432     CRM_ASSERT(lrmd != NULL);
 433 
 434     private = lrmd->private;
 435     switch (private->type) {
 436         case CRM_CLIENT_IPC:
 437             while (crm_ipc_ready(private->ipc)) {
 438                 if (crm_ipc_read(private->ipc) > 0) {
 439                     const char *msg = crm_ipc_buffer(private->ipc);
 440 
 441                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 442                 }
 443             }
 444             break;
 445 #ifdef HAVE_GNUTLS_GNUTLS_H
 446         case CRM_CLIENT_TLS:
 447             lrmd_tls_dispatch(lrmd);
 448             break;
 449 #endif
 450         default:
 451             crm_err("Unsupported connection type: %d", private->type);
 452     }
 453 
 454     if (lrmd_api_is_connected(lrmd) == FALSE) {
 455         crm_err("Connection closed");
 456         return FALSE;
 457     }
 458 
 459     return TRUE;
 460 }
 461 
 462 static xmlNode *
 463 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 464                enum lrmd_call_options options)
 465 {
 466     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 467 
 468     CRM_CHECK(op_msg != NULL, return NULL);
 469     CRM_CHECK(token != NULL, return NULL);
 470 
 471     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 472     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 473     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 474     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 475     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 476     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 477 
 478     if (data != NULL) {
 479         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 480     }
 481 
 482     crm_trace("Created lrmd %s command with call options %.8lx (%d)",
 483               op, (long)options, options);
 484     return op_msg;
 485 }
 486 
 487 static void
 488 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 489 {
 490     lrmd_t *lrmd = userdata;
 491     lrmd_private_t *native = lrmd->private;
 492 
 493     crm_info("IPC connection destroyed");
 494 
 495     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 496     native->ipc = NULL;
 497     native->source = NULL;
 498 
 499     if (native->callback) {
 500         lrmd_event_data_t event = { 0, };
 501         event.type = lrmd_event_disconnect;
 502         event.remote_nodename = native->remote_nodename;
 503         native->callback(&event);
 504     }
 505 }
 506 
 507 #ifdef HAVE_GNUTLS_GNUTLS_H
 508 static void
 509 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 510 {
 511     lrmd_t *lrmd = userdata;
 512     lrmd_private_t *native = lrmd->private;
 513 
 514     crm_info("TLS connection destroyed");
 515 
 516     if (native->remote->tls_session) {
 517         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 518         gnutls_deinit(*native->remote->tls_session);
 519         gnutls_free(native->remote->tls_session);
 520     }
 521     if (native->psk_cred_c) {
 522         gnutls_psk_free_client_credentials(native->psk_cred_c);
 523     }
 524     if (native->sock) {
 525         close(native->sock);
 526     }
 527     if (native->process_notify) {
 528         mainloop_destroy_trigger(native->process_notify);
 529         native->process_notify = NULL;
 530     }
 531     if (native->pending_notify) {
 532         g_list_free_full(native->pending_notify, lrmd_free_xml);
 533         native->pending_notify = NULL;
 534     }
 535 
 536     free(native->remote->buffer);
 537     native->remote->buffer = NULL;
 538     native->source = 0;
 539     native->sock = 0;
 540     native->psk_cred_c = NULL;
 541     native->remote->tls_session = NULL;
 542     native->sock = 0;
 543 
 544     if (native->callback) {
 545         lrmd_event_data_t event = { 0, };
 546         event.remote_nodename = native->remote_nodename;
 547         event.type = lrmd_event_disconnect;
 548         native->callback(&event);
 549     }
 550     return;
 551 }
 552 
 553 int
 554 lrmd_tls_send_msg(crm_remote_t * session, xmlNode * msg, uint32_t id, const char *msg_type)
     /*  */
 555 {
 556     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 557     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 558     return crm_remote_send(session, msg);
 559 }
 560 
 561 static xmlNode *
 562 lrmd_tls_recv_reply(lrmd_t * lrmd, int total_timeout, int expected_reply_id, int *disconnected)
     /*  */
 563 {
 564     lrmd_private_t *native = lrmd->private;
 565     xmlNode *xml = NULL;
 566     time_t start = time(NULL);
 567     const char *msg_type = NULL;
 568     int reply_id = 0;
 569     int remaining_timeout = 0;
 570 
 571     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 572      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 573     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 574         total_timeout = MAX_TLS_RECV_WAIT;
 575     }
 576 
 577     while (!xml) {
 578 
 579         xml = crm_remote_parse_buffer(native->remote);
 580         if (!xml) {
 581             /* read some more off the tls buffer if we still have time left. */
 582             if (remaining_timeout) {
 583                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 584             } else {
 585                 remaining_timeout = total_timeout;
 586             }
 587             if (remaining_timeout <= 0) {
 588                 crm_err("Never received the expected reply during the timeout period, disconnecting.");
 589                 *disconnected = TRUE;
 590                 return NULL;
 591             }
 592 
 593             crm_remote_recv(native->remote, remaining_timeout, disconnected);
 594             xml = crm_remote_parse_buffer(native->remote);
 595             if (!xml) {
 596                 crm_err("Unable to receive expected reply, disconnecting.");
 597                 *disconnected = TRUE;
 598                 return NULL;
 599             } else if (*disconnected) {
 600                 return NULL;
 601             }
 602         }
 603 
 604         CRM_ASSERT(xml != NULL);
 605 
 606         crm_element_value_int(xml, F_LRMD_REMOTE_MSG_ID, &reply_id);
 607         msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 608 
 609         if (!msg_type) {
 610             crm_err("Empty msg type received while waiting for reply");
 611             free_xml(xml);
 612             xml = NULL;
 613         } else if (safe_str_eq(msg_type, "notify")) {
 614             /* got a notify while waiting for reply, trigger the notify to be processed later */
 615             crm_info("queueing notify");
 616             native->pending_notify = g_list_append(native->pending_notify, xml);
 617             if (native->process_notify) {
 618                 crm_info("notify trigger set.");
 619                 mainloop_set_trigger(native->process_notify);
 620             }
 621             xml = NULL;
 622         } else if (safe_str_neq(msg_type, "reply")) {
 623             /* msg isn't a reply, make some noise */
 624             crm_err("Expected a reply, got %s", msg_type);
 625             free_xml(xml);
 626             xml = NULL;
 627         } else if (reply_id != expected_reply_id) {
 628             if (native->expected_late_replies > 0) {
 629                 native->expected_late_replies--;
 630             } else {
 631                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 632             }
 633             free_xml(xml);
 634             xml = NULL;
 635         }
 636     }
 637 
 638     if (native->remote->buffer && native->process_notify) {
 639         mainloop_set_trigger(native->process_notify);
 640     }
 641 
 642     return xml;
 643 }
 644 
 645 static int
 646 lrmd_tls_send(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 647 {
 648     int rc = 0;
 649     lrmd_private_t *native = lrmd->private;
 650 
 651     global_remote_msg_id++;
 652     if (global_remote_msg_id <= 0) {
 653         global_remote_msg_id = 1;
 654     }
 655 
 656     rc = lrmd_tls_send_msg(native->remote, msg, global_remote_msg_id, "request");
 657     if (rc <= 0) {
 658         crm_err("Remote lrmd send failed, disconnecting");
 659         lrmd_tls_disconnect(lrmd);
 660         return -ENOTCONN;
 661     }
 662     return pcmk_ok;
 663 }
 664 
 665 static int
 666 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 667 {
 668     int rc = 0;
 669     int disconnected = 0;
 670     xmlNode *xml = NULL;
 671 
 672     if (lrmd_tls_connected(lrmd) == FALSE) {
 673         return -1;
 674     }
 675 
 676     rc = lrmd_tls_send(lrmd, msg);
 677     if (rc < 0) {
 678         return rc;
 679     }
 680 
 681     xml = lrmd_tls_recv_reply(lrmd, timeout, global_remote_msg_id, &disconnected);
 682 
 683     if (disconnected) {
 684         crm_err("Remote lrmd server disconnected while waiting for reply with id %d. ",
 685                 global_remote_msg_id);
 686         lrmd_tls_disconnect(lrmd);
 687         rc = -ENOTCONN;
 688     } else if (!xml) {
 689         crm_err("Remote lrmd never received reply for request id %d. timeout: %dms ",
 690                 global_remote_msg_id, timeout);
 691         rc = -ECOMM;
 692     }
 693 
 694     if (reply) {
 695         *reply = xml;
 696     } else {
 697         free_xml(xml);
 698     }
 699 
 700     return rc;
 701 }
 702 #endif
 703 
 704 static int
 705 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 706 {
 707     int rc = -1;
 708     lrmd_private_t *native = lrmd->private;
 709 
 710     switch (native->type) {
 711         case CRM_CLIENT_IPC:
 712             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 713             break;
 714 #ifdef HAVE_GNUTLS_GNUTLS_H
 715         case CRM_CLIENT_TLS:
 716             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 717             break;
 718 #endif
 719         default:
 720             crm_err("Unsupported connection type: %d", native->type);
 721     }
 722 
 723     return rc;
 724 }
 725 
 726 static int
 727 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 728 {
 729     int rc = -1;
 730     lrmd_private_t *native = lrmd->private;
 731 
 732     switch (native->type) {
 733         case CRM_CLIENT_IPC:
 734             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 735             break;
 736 #ifdef HAVE_GNUTLS_GNUTLS_H
 737         case CRM_CLIENT_TLS:
 738             rc = lrmd_tls_send(lrmd, msg);
 739             if (rc == pcmk_ok) {
 740                 /* we don't want to wait around for the reply, but
 741                  * since the request/reply protocol needs to behave the same
 742                  * as libqb, a reply will eventually come later anyway. */
 743                 native->expected_late_replies++;
 744             }
 745             break;
 746 #endif
 747         default:
 748             crm_err("Unsupported connection type: %d", native->type);
 749     }
 750 
 751     return rc;
 752 }
 753 
 754 static int
 755 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 756 {
 757     lrmd_private_t *native = lrmd->private;
 758 
 759     switch (native->type) {
 760         case CRM_CLIENT_IPC:
 761             return crm_ipc_connected(native->ipc);
 762             break;
 763 #ifdef HAVE_GNUTLS_GNUTLS_H
 764         case CRM_CLIENT_TLS:
 765             return lrmd_tls_connected(lrmd);
 766             break;
 767 #endif
 768         default:
 769             crm_err("Unsupported connection type: %d", native->type);
 770     }
 771 
 772     return 0;
 773 }
 774 
 775 /*!
 776  * \internal
 777  * \brief Send a prepared API command to the lrmd server
 778  *
 779  * \param[in]  lrmd          Existing connection to the lrmd server
 780  * \param[in]  op            Name of API command to send
 781  * \param[in]  data          Command data XML to add to the sent command
 782  * \param[out] output_data   If expecting a reply, it will be stored here
 783  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to 1000);
 784  *                           will be added to the command XML
 785  * \param[in]  call_options  Call options to pass to server when sending
 786  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
 787  *                           must be TRUE for IPC (as opposed to TLS) clients
 788  *
 789  * \return pcmk_ok on success, -errno on error
 790  */
 791 static int
 792 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 793                   xmlNode **output_data, int timeout,
 794                   enum lrmd_call_options options, gboolean expect_reply)
 795 {
 796     int rc = pcmk_ok;
 797     lrmd_private_t *native = lrmd->private;
 798     xmlNode *op_msg = NULL;
 799     xmlNode *op_reply = NULL;
 800 
 801     if (!lrmd_api_is_connected(lrmd)) {
 802         return -ENOTCONN;
 803     }
 804 
 805     if (op == NULL) {
 806         crm_err("No operation specified");
 807         return -EINVAL;
 808     }
 809 
 810     CRM_CHECK(native->token != NULL,;
 811         );
 812     crm_trace("sending %s op to lrmd", op);
 813 
 814     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 815 
 816     if (op_msg == NULL) {
 817         return -EINVAL;
 818     }
 819 
 820     if (expect_reply) {
 821         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 822     } else {
 823         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 824         goto done;
 825     }
 826 
 827     if (rc < 0) {
 828         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 829         rc = -ECOMM;
 830         goto done;
 831 
 832     } else if(op_reply == NULL) {
 833         rc = -ENOMSG;
 834         goto done;
 835     }
 836 
 837     rc = pcmk_ok;
 838     crm_trace("%s op reply received", op);
 839     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 840         rc = -ENOMSG;
 841         goto done;
 842     }
 843 
 844     crm_log_xml_trace(op_reply, "Reply");
 845 
 846     if (output_data) {
 847         *output_data = op_reply;
 848         op_reply = NULL;        /* Prevent subsequent free */
 849     }
 850 
 851   done:
 852     if (lrmd_api_is_connected(lrmd) == FALSE) {
 853         crm_err("LRMD disconnected");
 854     }
 855 
 856     free_xml(op_msg);
 857     free_xml(op_reply);
 858     return rc;
 859 }
 860 
 861 static int
 862 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 863 {
 864     int rc;
 865     lrmd_private_t *native = lrmd->private;
 866     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 867 
 868     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
 869     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0, native->type == CRM_CLIENT_IPC ? TRUE : FALSE);
 870     free_xml(data);
 871 
 872     return rc < 0 ? rc : pcmk_ok;
 873 }
 874 
 875 int
 876 remote_proxy_check(lrmd_t * lrmd, GHashTable *hash)
     /*  */
 877 {
 878     int rc;
 879     const char *value;
 880     lrmd_private_t *native = lrmd->private;
 881     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 882 
 883     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
 884 
 885     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 886     crm_xml_add(data, F_LRMD_WATCHDOG, value);
 887 
 888     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0, native->type == CRM_CLIENT_IPC ? TRUE : FALSE);
 889     free_xml(data);
 890 
 891     return rc < 0 ? rc : pcmk_ok;
 892 }
 893 
 894 static int
 895 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 896 {
 897     int rc = pcmk_ok;
 898     lrmd_private_t *native = lrmd->private;
 899     xmlNode *reply = NULL;
 900     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 901 
 902     crm_xml_add(hello, F_TYPE, T_LRMD);
 903     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 904     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 905     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 906 
 907     /* advertise that we are a proxy provider */
 908     if (native->proxy_callback) {
 909         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
 910     }
 911 
 912     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 913 
 914     if (rc < 0) {
 915         crm_perror(LOG_DEBUG, "Couldn't complete registration with the lrmd API: %d", rc);
 916         rc = -ECOMM;
 917     } else if (reply == NULL) {
 918         crm_err("Did not receive registration reply");
 919         rc = -EPROTO;
 920     } else {
 921         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 922         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 923         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 924 
 925         crm_element_value_int(reply, F_LRMD_RC, &rc);
 926 
 927         if (rc == -EPROTO) {
 928             crm_err("LRMD protocol mismatch client version %s, server version %s",
 929                 LRMD_PROTOCOL_VERSION, version);
 930             crm_log_xml_err(reply, "Protocol Error");
 931 
 932         } else if (safe_str_neq(msg_type, CRM_OP_REGISTER)) {
 933             crm_err("Invalid registration message: %s", msg_type);
 934             crm_log_xml_err(reply, "Bad reply");
 935             rc = -EPROTO;
 936         } else if (tmp_ticket == NULL) {
 937             crm_err("No registration token provided");
 938             crm_log_xml_err(reply, "Bad reply");
 939             rc = -EPROTO;
 940         } else {
 941             crm_trace("Obtained registration token: %s", tmp_ticket);
 942             native->token = strdup(tmp_ticket);
 943             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
 944             rc = pcmk_ok;
 945         }
 946     }
 947 
 948     free_xml(reply);
 949     free_xml(hello);
 950 
 951     if (rc != pcmk_ok) {
 952         lrmd_api_disconnect(lrmd);
 953     }
 954     return rc;
 955 }
 956 
 957 static int
 958 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
 959 {
 960     int rc = pcmk_ok;
 961     lrmd_private_t *native = lrmd->private;
 962 
 963     static struct ipc_client_callbacks lrmd_callbacks = {
 964         .dispatch = lrmd_ipc_dispatch,
 965         .destroy = lrmd_ipc_connection_destroy
 966     };
 967 
 968     crm_info("Connecting to lrmd");
 969 
 970     if (fd) {
 971         /* No mainloop */
 972         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
 973         if (native->ipc && crm_ipc_connect(native->ipc)) {
 974             *fd = crm_ipc_get_fd(native->ipc);
 975         } else if (native->ipc) {
 976             crm_perror(LOG_ERR, "Connection to local resource manager failed");
 977             rc = -ENOTCONN;
 978         }
 979     } else {
 980         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
 981         native->ipc = mainloop_get_ipc_client(native->source);
 982     }
 983 
 984     if (native->ipc == NULL) {
 985         crm_debug("Could not connect to the LRMD API");
 986         rc = -ENOTCONN;
 987     }
 988 
 989     return rc;
 990 }
 991 
 992 #ifdef HAVE_GNUTLS_GNUTLS_H
 993 static int
 994 set_key(gnutls_datum_t * key, const char *location)
     /*  */
 995 {
 996     FILE *stream;
 997     int read_len = 256;
 998     int cur_len = 0;
 999     int buf_len = read_len;
1000     static char *key_cache = NULL;
1001     static size_t key_cache_len = 0;
1002     static time_t key_cache_updated;
1003 
1004     if (location == NULL) {
1005         return -1;
1006     }
1007 
1008     if (key_cache) {
1009         time_t now = time(NULL);
1010 
1011         if ((now - key_cache_updated) < 60) {
1012             key->data = gnutls_malloc(key_cache_len + 1);
1013             key->size = key_cache_len;
1014             memcpy(key->data, key_cache, key_cache_len);
1015 
1016             crm_debug("using cached LRMD key");
1017             return 0;
1018         } else {
1019             key_cache_len = 0;
1020             key_cache_updated = 0;
1021             free(key_cache);
1022             key_cache = NULL;
1023             crm_debug("clearing lrmd key cache");
1024         }
1025     }
1026 
1027     stream = fopen(location, "r");
1028     if (!stream) {
1029         return -1;
1030     }
1031 
1032     key->data = gnutls_malloc(read_len);
1033     while (!feof(stream)) {
1034         int next;
1035 
1036         if (cur_len == buf_len) {
1037             buf_len = cur_len + read_len;
1038             key->data = gnutls_realloc(key->data, buf_len);
1039         }
1040         next = fgetc(stream);
1041         if (next == EOF && feof(stream)) {
1042             break;
1043         }
1044 
1045         key->data[cur_len] = next;
1046         cur_len++;
1047     }
1048     fclose(stream);
1049 
1050     key->size = cur_len;
1051     if (!cur_len) {
1052         gnutls_free(key->data);
1053         key->data = 0;
1054         return -1;
1055     }
1056 
1057     if (!key_cache) {
1058         key_cache = calloc(1, key->size + 1);
1059         memcpy(key_cache, key->data, key->size);
1060 
1061         key_cache_len = key->size;
1062         key_cache_updated = time(NULL);
1063     }
1064 
1065     return 0;
1066 }
1067 
1068 int
1069 lrmd_tls_set_key(gnutls_datum_t * key)
     /*  */
1070 {
1071     const char *specific_location = getenv("PCMK_authkey_location");
1072 
1073     if (set_key(key, specific_location) == 0) {
1074         crm_debug("Using custom authkey location %s", specific_location);
1075         return pcmk_ok;
1076 
1077     } else if (specific_location) {
1078         crm_err("No valid lrmd remote key found at %s, trying default location", specific_location);
1079     }
1080 
1081     if ((set_key(key, DEFAULT_REMOTE_KEY_LOCATION) != 0)
1082         && (set_key(key, ALT_REMOTE_KEY_LOCATION) != 0)) {
1083         crm_err("No valid lrmd remote key found at %s", DEFAULT_REMOTE_KEY_LOCATION);
1084         return -ENOKEY;
1085     }
1086 
1087     return pcmk_ok;
1088 }
1089 
1090 static void
1091 lrmd_gnutls_global_init(void)
     /*  */
1092 {
1093     static int gnutls_init = 0;
1094 
1095     if (!gnutls_init) {
1096         crm_gnutls_global_init();
1097     }
1098     gnutls_init = 1;
1099 }
1100 #endif
1101 
1102 static void
1103 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1104 {
1105     lrmd_private_t *native = lrmd->private;
1106 
1107     if (native->callback) {
1108         lrmd_event_data_t event = { 0, };
1109         event.type = lrmd_event_connect;
1110         event.remote_nodename = native->remote_nodename;
1111         event.connection_rc = rc;
1112         native->callback(&event);
1113     }
1114 }
1115 
1116 #ifdef HAVE_GNUTLS_GNUTLS_H
1117 static void
1118 lrmd_tcp_connect_cb(void *userdata, int sock)
     /*  */
1119 {
1120     lrmd_t *lrmd = userdata;
1121     lrmd_private_t *native = lrmd->private;
1122     char *name;
1123     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
1124         .dispatch = lrmd_tls_dispatch,
1125         .destroy = lrmd_tls_connection_destroy,
1126     };
1127     int rc = sock;
1128     gnutls_datum_t psk_key = { NULL, 0 };
1129 
1130     native->async_timer = 0;
1131 
1132     if (rc < 0) {
1133         lrmd_tls_connection_destroy(lrmd);
1134         crm_info("Could not connect to remote LRMD at %s:%d",
1135                  native->server, native->port);
1136         report_async_connection_result(lrmd, rc);
1137         return;
1138     }
1139 
1140     /* The TCP connection was successful, so establish the TLS connection.
1141      * @TODO make this async to avoid blocking code in client
1142      */
1143 
1144     native->sock = sock;
1145 
1146     rc = lrmd_tls_set_key(&psk_key);
1147     if (rc != 0) {
1148         crm_warn("Could not set key for remote LRMD at %s:%d " CRM_XS " rc=%d",
1149                  native->server, native->port, rc);
1150         lrmd_tls_connection_destroy(lrmd);
1151         report_async_connection_result(lrmd, rc);
1152         return;
1153     }
1154 
1155     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1156     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1157     gnutls_free(psk_key.data);
1158 
1159     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
1160 
1161     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
1162         crm_warn("Disconnecting after TLS handshake with remote LRMD %s:%d failed",
1163                  native->server, native->port);
1164         gnutls_deinit(*native->remote->tls_session);
1165         gnutls_free(native->remote->tls_session);
1166         native->remote->tls_session = NULL;
1167         lrmd_tls_connection_destroy(lrmd);
1168         report_async_connection_result(lrmd, -EKEYREJECTED);
1169         return;
1170     }
1171 
1172     crm_info("TLS connection to remote LRMD %s:%d succeeded",
1173              native->server, native->port);
1174 
1175     name = crm_strdup_printf("remote-lrmd-%s:%d", native->server, native->port);
1176 
1177     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
1178     native->source =
1179         mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
1180 
1181     rc = lrmd_handshake(lrmd, name);
1182     free(name);
1183 
1184     report_async_connection_result(lrmd, rc);
1185     return;
1186 }
1187 
1188 static int
1189 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1190 {
1191     int sock = 0;
1192     int timer_id = 0;
1193     lrmd_private_t *native = lrmd->private;
1194 
1195     lrmd_gnutls_global_init();
1196     sock = crm_remote_tcp_connect_async(native->server, native->port, timeout,
1197                                         &timer_id, lrmd, lrmd_tcp_connect_cb);
1198     if (sock < 0) {
1199         return sock;
1200     }
1201     native->sock = sock;
1202     native->async_timer = timer_id;
1203     return pcmk_ok;
1204 }
1205 
1206 static int
1207 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1208 {
1209     static struct mainloop_fd_callbacks lrmd_tls_callbacks = {
1210         .dispatch = lrmd_tls_dispatch,
1211         .destroy = lrmd_tls_connection_destroy,
1212     };
1213     int rc;
1214 
1215     lrmd_private_t *native = lrmd->private;
1216     int sock;
1217     gnutls_datum_t psk_key = { NULL, 0 };
1218 
1219     lrmd_gnutls_global_init();
1220 
1221     sock = crm_remote_tcp_connect(native->server, native->port);
1222     if (sock < 0) {
1223         crm_warn("Could not establish remote lrmd connection to %s", native->server);
1224         lrmd_tls_connection_destroy(lrmd);
1225         return -ENOTCONN;
1226     }
1227 
1228     native->sock = sock;
1229 
1230     rc = lrmd_tls_set_key(&psk_key);
1231     if (rc < 0) {
1232         lrmd_tls_connection_destroy(lrmd);
1233         return rc;
1234     }
1235 
1236     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1237     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1238     gnutls_free(psk_key.data);
1239 
1240     native->remote->tls_session = create_psk_tls_session(sock, GNUTLS_CLIENT, native->psk_cred_c);
1241 
1242     if (crm_initiate_client_tls_handshake(native->remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT) != 0) {
1243         crm_err("Session creation for %s:%d failed", native->server, native->port);
1244         gnutls_deinit(*native->remote->tls_session);
1245         gnutls_free(native->remote->tls_session);
1246         native->remote->tls_session = NULL;
1247         lrmd_tls_connection_destroy(lrmd);
1248         return -EKEYREJECTED;
1249     }
1250 
1251     crm_info("Remote lrmd client TLS connection established with server %s:%d", native->server,
1252              native->port);
1253 
1254     if (fd) {
1255         *fd = sock;
1256     } else {
1257         char *name = crm_strdup_printf("remote-lrmd-%s:%d",
1258                                        native->server, native->port);
1259 
1260         native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_tls_dispatch, lrmd);
1261         native->source =
1262             mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd, &lrmd_tls_callbacks);
1263         free(name);
1264     }
1265     return pcmk_ok;
1266 }
1267 #endif
1268 
1269 static int
1270 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1271 {
1272     int rc = -ENOTCONN;
1273     lrmd_private_t *native = lrmd->private;
1274 
1275     switch (native->type) {
1276         case CRM_CLIENT_IPC:
1277             rc = lrmd_ipc_connect(lrmd, fd);
1278             break;
1279 #ifdef HAVE_GNUTLS_GNUTLS_H
1280         case CRM_CLIENT_TLS:
1281             rc = lrmd_tls_connect(lrmd, fd);
1282             break;
1283 #endif
1284         default:
1285             crm_err("Unsupported connection type: %d", native->type);
1286     }
1287 
1288     if (rc == pcmk_ok) {
1289         rc = lrmd_handshake(lrmd, name);
1290     }
1291 
1292     return rc;
1293 }
1294 
1295 static int
1296 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1297 {
1298     int rc = 0;
1299     lrmd_private_t *native = lrmd->private;
1300 
1301     if (!native->callback) {
1302         crm_err("Async connect not possible, no lrmd client callback set.");
1303         return -1;
1304     }
1305 
1306     switch (native->type) {
1307         case CRM_CLIENT_IPC:
1308             /* fake async connection with ipc.  it should be fast
1309              * enough that we gain very little from async */
1310             rc = lrmd_api_connect(lrmd, name, NULL);
1311             if (!rc) {
1312                 report_async_connection_result(lrmd, rc);
1313             }
1314             break;
1315 #ifdef HAVE_GNUTLS_GNUTLS_H
1316         case CRM_CLIENT_TLS:
1317             rc = lrmd_tls_connect_async(lrmd, timeout);
1318             if (rc) {
1319                 /* connection failed, report rc now */
1320                 report_async_connection_result(lrmd, rc);
1321             }
1322             break;
1323 #endif
1324         default:
1325             crm_err("Unsupported connection type: %d", native->type);
1326     }
1327 
1328     return rc;
1329 }
1330 
1331 static void
1332 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1333 {
1334     lrmd_private_t *native = lrmd->private;
1335 
1336     if (native->source != NULL) {
1337         /* Attached to mainloop */
1338         mainloop_del_ipc_client(native->source);
1339         native->source = NULL;
1340         native->ipc = NULL;
1341 
1342     } else if (native->ipc) {
1343         /* Not attached to mainloop */
1344         crm_ipc_t *ipc = native->ipc;
1345 
1346         native->ipc = NULL;
1347         crm_ipc_close(ipc);
1348         crm_ipc_destroy(ipc);
1349     }
1350 }
1351 
1352 #ifdef HAVE_GNUTLS_GNUTLS_H
1353 static void
1354 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1355 {
1356     lrmd_private_t *native = lrmd->private;
1357 
1358     if (native->remote->tls_session) {
1359         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1360         gnutls_deinit(*native->remote->tls_session);
1361         gnutls_free(native->remote->tls_session);
1362         native->remote->tls_session = 0;
1363     }
1364 
1365     if (native->async_timer) {
1366         g_source_remove(native->async_timer);
1367         native->async_timer = 0;
1368     }
1369 
1370     if (native->source != NULL) {
1371         /* Attached to mainloop */
1372         mainloop_del_ipc_client(native->source);
1373         native->source = NULL;
1374 
1375     } else if (native->sock) {
1376         close(native->sock);
1377         native->sock = 0;
1378     }
1379 
1380     if (native->pending_notify) {
1381         g_list_free_full(native->pending_notify, lrmd_free_xml);
1382         native->pending_notify = NULL;
1383     }
1384 }
1385 #endif
1386 
1387 static int
1388 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1389 {
1390     lrmd_private_t *native = lrmd->private;
1391 
1392     crm_info("Disconnecting from %d lrmd service", native->type);
1393     switch (native->type) {
1394         case CRM_CLIENT_IPC:
1395             lrmd_ipc_disconnect(lrmd);
1396             break;
1397 #ifdef HAVE_GNUTLS_GNUTLS_H
1398         case CRM_CLIENT_TLS:
1399             lrmd_tls_disconnect(lrmd);
1400             break;
1401 #endif
1402         default:
1403             crm_err("Unsupported connection type: %d", native->type);
1404     }
1405 
1406     free(native->token);
1407     native->token = NULL;
1408 
1409     free(native->peer_version);
1410     native->peer_version = NULL;
1411     return 0;
1412 }
1413 
1414 static int
1415 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1416                       const char *rsc_id,
1417                       const char *class,
1418                       const char *provider, const char *type, enum lrmd_call_options options)
1419 {
1420     int rc = pcmk_ok;
1421     xmlNode *data = NULL;
1422 
1423     if (!class || !type || !rsc_id) {
1424         return -EINVAL;
1425     }
1426     if (crm_provider_required(class) && !provider) {
1427         return -EINVAL;
1428     }
1429 
1430     data = create_xml_node(NULL, F_LRMD_RSC);
1431 
1432     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1433     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1434     crm_xml_add(data, F_LRMD_CLASS, class);
1435     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1436     crm_xml_add(data, F_LRMD_TYPE, type);
1437     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1438     free_xml(data);
1439 
1440     return rc;
1441 }
1442 
1443 static int
1444 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1445 {
1446     int rc = pcmk_ok;
1447     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1448 
1449     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1450     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1451     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1452     free_xml(data);
1453 
1454     return rc;
1455 }
1456 
1457 lrmd_rsc_info_t *
1458 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1459 {
1460     lrmd_rsc_info_t *copy = NULL;
1461 
1462     copy = calloc(1, sizeof(lrmd_rsc_info_t));
1463 
1464     copy->id = strdup(rsc_info->id);
1465     copy->type = strdup(rsc_info->type);
1466     copy->class = strdup(rsc_info->class);
1467     if (rsc_info->provider) {
1468         copy->provider = strdup(rsc_info->provider);
1469     }
1470 
1471     return copy;
1472 }
1473 
1474 void
1475 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1476 {
1477     if (!rsc_info) {
1478         return;
1479     }
1480     free(rsc_info->id);
1481     free(rsc_info->type);
1482     free(rsc_info->class);
1483     free(rsc_info->provider);
1484     free(rsc_info);
1485 }
1486 
1487 static lrmd_rsc_info_t *
1488 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1489 {
1490     lrmd_rsc_info_t *rsc_info = NULL;
1491     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1492     xmlNode *output = NULL;
1493     const char *class = NULL;
1494     const char *provider = NULL;
1495     const char *type = NULL;
1496 
1497     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1498     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1499     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1500     free_xml(data);
1501 
1502     if (!output) {
1503         return NULL;
1504     }
1505 
1506     class = crm_element_value(output, F_LRMD_CLASS);
1507     provider = crm_element_value(output, F_LRMD_PROVIDER);
1508     type = crm_element_value(output, F_LRMD_TYPE);
1509 
1510     if (!class || !type) {
1511         free_xml(output);
1512         return NULL;
1513     } else if (crm_provider_required(class) && !provider) {
1514         free_xml(output);
1515         return NULL;
1516     }
1517 
1518     rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1519     rsc_info->id = strdup(rsc_id);
1520     rsc_info->class = strdup(class);
1521     if (provider) {
1522         rsc_info->provider = strdup(provider);
1523     }
1524     rsc_info->type = strdup(type);
1525 
1526     free_xml(output);
1527     return rsc_info;
1528 }
1529 
1530 static void
1531 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1532 {
1533     lrmd_private_t *native = lrmd->private;
1534 
1535     native->callback = callback;
1536 }
1537 
1538 void
1539 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1540 {
1541     lrmd_private_t *native = lrmd->private;
1542 
1543     native->proxy_callback = callback;
1544     native->proxy_callback_userdata = userdata;
1545 }
1546 
1547 void
1548 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1549 {
1550     lrmd_private_t *native = lrmd->private;
1551 
1552     if (native->proxy_callback) {
1553         crm_log_xml_trace(msg, "PROXY_INBOUND");
1554         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1555     }
1556 }
1557 
1558 int
1559 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1560 {
1561     if (lrmd == NULL) {
1562         return -ENOTCONN;
1563     }
1564     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1565 
1566     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1567     return lrmd_send_xml_no_reply(lrmd, msg);
1568 }
1569 
1570 static int
1571 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1572 {
1573     int rc = pcmk_ok;
1574     stonith_t *stonith_api = stonith_api_new();
1575 
1576     if(stonith_api) {
1577         stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type, provider, output, 0);
1578         stonith_api->cmds->free(stonith_api);
1579     }
1580     if (*output == NULL) {
1581         rc = -EIO;
1582     }
1583     return rc;
1584 }
1585 
1586 static int
1587 lrmd_api_get_metadata(lrmd_t * lrmd,
     /*  */
1588                       const char *class,
1589                       const char *provider,
1590                       const char *type, char **output, enum lrmd_call_options options)
1591 {
1592     svc_action_t *action;
1593 
1594     if (!class || !type) {
1595         return -EINVAL;
1596     }
1597 
1598     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
1599         return stonith_get_metadata(provider, type, output);
1600     }
1601 
1602     action = resources_action_create(type, class, provider, type,
1603                                      "meta-data", 0,
1604                                      CRMD_METADATA_CALL_TIMEOUT, NULL, 0);
1605     if (action == NULL) {
1606         crm_err("Unable to retrieve meta-data for %s:%s:%s", class, provider, type);
1607         services_action_free(action);
1608         return -EINVAL;
1609     }
1610 
1611     if (!(services_action_sync(action))) {
1612         crm_err("Failed to retrieve meta-data for %s:%s:%s", class, provider, type);
1613         services_action_free(action);
1614         return -EIO;
1615     }
1616 
1617     if (!action->stdout_data) {
1618         crm_err("Failed to receive meta-data for %s:%s:%s", class, provider, type);
1619         services_action_free(action);
1620         return -EIO;
1621     }
1622 
1623     *output = strdup(action->stdout_data);
1624     services_action_free(action);
1625 
1626     return pcmk_ok;
1627 }
1628 
1629 static int
1630 lrmd_api_exec(lrmd_t * lrmd, const char *rsc_id, const char *action, const char *userdata, int interval,        /* ms */
     /*  */
1631               int timeout,      /* ms */
1632               int start_delay,  /* ms */
1633               enum lrmd_call_options options, lrmd_key_value_t * params)
1634 {
1635     int rc = pcmk_ok;
1636     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1637     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
1638     lrmd_key_value_t *tmp = NULL;
1639 
1640     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1641     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1642     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
1643     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
1644     crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
1645     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
1646     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
1647 
1648     for (tmp = params; tmp; tmp = tmp->next) {
1649         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
1650     }
1651 
1652     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
1653     free_xml(data);
1654 
1655     lrmd_key_value_freeall(params);
1656     return rc;
1657 }
1658 
1659 /* timeout is in ms */
1660 static int
1661 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
1662                     int timeout, lrmd_key_value_t *params)
1663 {
1664     int rc = pcmk_ok;
1665     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
1666     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
1667     lrmd_key_value_t *tmp = NULL;
1668 
1669     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1670     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
1671     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
1672     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
1673 
1674     for (tmp = params; tmp; tmp = tmp->next) {
1675         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
1676     }
1677 
1678     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
1679                            lrmd_opt_notify_orig_only, TRUE);
1680     free_xml(data);
1681 
1682     lrmd_key_value_freeall(params);
1683     return rc;
1684 }
1685 
1686 static int
1687 lrmd_api_cancel(lrmd_t * lrmd, const char *rsc_id, const char *action, int interval)
     /*  */
1688 {
1689     int rc = pcmk_ok;
1690     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1691 
1692     crm_xml_add(data, F_LRMD_ORIGIN, __FUNCTION__);
1693     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
1694     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1695     crm_xml_add_int(data, F_LRMD_RSC_INTERVAL, interval);
1696     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
1697     free_xml(data);
1698     return rc;
1699 }
1700 
1701 static int
1702 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
1703 {
1704     int rc = 0;
1705     stonith_t *stonith_api = stonith_api_new();
1706     stonith_key_value_t *stonith_resources = NULL;
1707     stonith_key_value_t *dIter = NULL;
1708 
1709     if(stonith_api) {
1710         stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL, &stonith_resources, 0);
1711         stonith_api->cmds->free(stonith_api);
1712     }
1713 
1714     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
1715         rc++;
1716         if (resources) {
1717             *resources = lrmd_list_add(*resources, dIter->value);
1718         }
1719     }
1720 
1721     stonith_key_value_freeall(stonith_resources, 1, 0);
1722     return rc;
1723 }
1724 
1725 static int
1726 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
1727                      const char *provider)
1728 {
1729     int rc = 0;
1730 
1731     if (safe_str_eq(class, PCMK_RESOURCE_CLASS_STONITH)) {
1732         rc += list_stonith_agents(resources);
1733 
1734     } else {
1735         GListPtr gIter = NULL;
1736         GList *agents = resources_list_agents(class, provider);
1737 
1738         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
1739             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
1740             rc++;
1741         }
1742         g_list_free_full(agents, free);
1743 
1744         if (!class) {
1745             rc += list_stonith_agents(resources);
1746         }
1747     }
1748 
1749     if (rc == 0) {
1750         crm_notice("No agents found for class %s", class);
1751         rc = -EPROTONOSUPPORT;
1752     }
1753     return rc;
1754 }
1755 
1756 static int
1757 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
1758 {
1759     int found = 0;
1760     GList *agents = NULL;
1761     GListPtr gIter2 = NULL;
1762 
1763     agents = resources_list_agents(class, provider);
1764     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
1765         if (safe_str_eq(agent, gIter2->data)) {
1766             found = 1;
1767         }
1768     }
1769     g_list_free_full(agents, free);
1770 
1771     return found;
1772 }
1773 
1774 static int
1775 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
1776 {
1777     int rc = pcmk_ok;
1778     char *provider = NULL;
1779     GList *ocf_providers = NULL;
1780     GListPtr gIter = NULL;
1781 
1782     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
1783 
1784     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
1785         provider = gIter->data;
1786         if (!agent || does_provider_have_agent(agent, provider,
1787                                                PCMK_RESOURCE_CLASS_OCF)) {
1788             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
1789             rc++;
1790         }
1791     }
1792 
1793     g_list_free_full(ocf_providers, free);
1794     return rc;
1795 }
1796 
1797 static int
1798 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
1799 {
1800     int rc = 0;
1801     GList *standards = NULL;
1802     GListPtr gIter = NULL;
1803 
1804     standards = resources_list_standards();
1805 
1806     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
1807         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
1808         rc++;
1809     }
1810 
1811     if (list_stonith_agents(NULL) > 0) {
1812         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
1813         rc++;
1814     }
1815 
1816     g_list_free_full(standards, free);
1817     return rc;
1818 }
1819 
1820 lrmd_t *
1821 lrmd_api_new(void)
     /*  */
1822 {
1823     lrmd_t *new_lrmd = NULL;
1824     lrmd_private_t *pvt = NULL;
1825 
1826     new_lrmd = calloc(1, sizeof(lrmd_t));
1827     pvt = calloc(1, sizeof(lrmd_private_t));
1828     pvt->remote = calloc(1, sizeof(crm_remote_t));
1829     new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));
1830 
1831     pvt->type = CRM_CLIENT_IPC;
1832     new_lrmd->private = pvt;
1833 
1834     new_lrmd->cmds->connect = lrmd_api_connect;
1835     new_lrmd->cmds->connect_async = lrmd_api_connect_async;
1836     new_lrmd->cmds->is_connected = lrmd_api_is_connected;
1837     new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
1838     new_lrmd->cmds->disconnect = lrmd_api_disconnect;
1839     new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
1840     new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
1841     new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
1842     new_lrmd->cmds->set_callback = lrmd_api_set_callback;
1843     new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
1844     new_lrmd->cmds->exec = lrmd_api_exec;
1845     new_lrmd->cmds->cancel = lrmd_api_cancel;
1846     new_lrmd->cmds->list_agents = lrmd_api_list_agents;
1847     new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
1848     new_lrmd->cmds->list_standards = lrmd_api_list_standards;
1849     new_lrmd->cmds->exec_alert = lrmd_api_exec_alert;
1850 
1851     return new_lrmd;
1852 }
1853 
1854 lrmd_t *
1855 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
1856 {
1857 #ifdef HAVE_GNUTLS_GNUTLS_H
1858     lrmd_t *new_lrmd = lrmd_api_new();
1859     lrmd_private_t *native = new_lrmd->private;
1860 
1861     if (!nodename && !server) {
1862         lrmd_api_delete(new_lrmd);
1863         return NULL;
1864     }
1865 
1866     native->type = CRM_CLIENT_TLS;
1867     native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
1868     native->server = server ? strdup(server) : strdup(nodename);
1869     native->port = port;
1870     if (native->port == 0) {
1871         native->port = crm_default_remote_port();
1872     }
1873 
1874     return new_lrmd;
1875 #else
1876     crm_err("GNUTLS is not enabled for this build, remote LRMD client can not be created");
1877     return NULL;
1878 #endif
1879 
1880 }
1881 
1882 void
1883 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
1884 {
1885     if (!lrmd) {
1886         return;
1887     }
1888     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
1889     free(lrmd->cmds);
1890     if (lrmd->private) {
1891         lrmd_private_t *native = lrmd->private;
1892 
1893 #ifdef HAVE_GNUTLS_GNUTLS_H
1894         free(native->server);
1895 #endif
1896         free(native->remote_nodename);
1897         free(native->remote);
1898         free(native->token);
1899         free(native->peer_version);
1900     }
1901 
1902     free(lrmd->private);
1903     free(lrmd);
1904 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
