lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
remote_executor_connected
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd__remote_send_xml
read_remote_reply
send_remote_message
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
lrmd__validate_remote_settings
lrmd_handshake
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
read_gnutls_key
key_is_cached
key_cache_expired
clear_key_cache
get_cached_key
cache_key
get_remote_key
lrmd__init_remote_key
lrmd_gnutls_global_init
report_async_connection_result
tls_client_handshake
add_tls_to_mainloop
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd__new
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
metadata_complete
lrmd__metadata_async
lrmd__set_result
lrmd__reset_result
lrmd__uptime
lrmd__node_start_state
   1 /*
   2  * Copyright 2012-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/common/xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>   // stonith__*
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  include <gnutls/gnutls.h>
  42 #endif
  43 
  44 #include <sys/socket.h>
  45 #include <netinet/in.h>
  46 #include <netinet/ip.h>
  47 #include <arpa/inet.h>
  48 #include <netdb.h>
  49 
  50 #define MAX_TLS_RECV_WAIT 10000
  51 
  52 CRM_TRACE_INIT_DATA(lrmd);
  53 
  54 static int lrmd_api_disconnect(lrmd_t * lrmd);
  55 static int lrmd_api_is_connected(lrmd_t * lrmd);
  56 
  57 /* IPC proxy functions */
  58 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  59 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  60 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  61 
  62 #ifdef HAVE_GNUTLS_GNUTLS_H
  63 // GnuTLS client handshake timeout in seconds
  64 #define TLS_HANDSHAKE_TIMEOUT 5
  65 
  66 gnutls_psk_client_credentials_t psk_cred_s;
  67 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  68 static int global_remote_msg_id = 0;
  69 static void lrmd_tls_connection_destroy(gpointer userdata);
  70 #endif
  71 
  72 typedef struct lrmd_private_s {
  73     uint64_t type;
  74     char *token;
  75     mainloop_io_t *source;
  76 
  77     /* IPC parameters */
  78     crm_ipc_t *ipc;
  79 
  80     pcmk__remote_t *remote;
  81 
  82     /* Extra TLS parameters */
  83     char *remote_nodename;
  84 #ifdef HAVE_GNUTLS_GNUTLS_H
  85     char *server;
  86     int port;
  87     gnutls_psk_client_credentials_t psk_cred_c;
  88 
  89     /* while the async connection is occurring, this is the id
  90      * of the connection timeout timer. */
  91     int async_timer;
  92     int sock;
  93     /* since tls requires a round trip across the network for a
  94      * request/reply, there are times where we just want to be able
  95      * to send a request from the client and not wait around (or even care
  96      * about) what the reply is. */
  97     int expected_late_replies;
  98     GList *pending_notify;
  99     crm_trigger_t *process_notify;
 100 #endif
 101 
 102     lrmd_event_callback callback;
 103 
 104     /* Internal IPC proxy msg passing for remote guests */
 105     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 106     void *proxy_callback_userdata;
 107     char *peer_version;
 108 } lrmd_private_t;
 109 
 110 static lrmd_list_t *
 111 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 112 {
 113     lrmd_list_t *p, *end;
 114 
 115     p = pcmk__assert_alloc(1, sizeof(lrmd_list_t));
 116     p->val = strdup(value);
 117 
 118     end = head;
 119     while (end && end->next) {
 120         end = end->next;
 121     }
 122 
 123     if (end) {
 124         end->next = p;
 125     } else {
 126         head = p;
 127     }
 128 
 129     return head;
 130 }
 131 
 132 void
 133 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 134 {
 135     lrmd_list_t *p;
 136 
 137     while (head) {
 138         char *val = (char *)head->val;
 139 
 140         p = head->next;
 141         free(val);
 142         free(head);
 143         head = p;
 144     }
 145 }
 146 
 147 lrmd_key_value_t *
 148 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 149 {
 150     lrmd_key_value_t *p, *end;
 151 
 152     p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t));
 153     p->key = strdup(key);
 154     p->value = strdup(value);
 155 
 156     end = head;
 157     while (end && end->next) {
 158         end = end->next;
 159     }
 160 
 161     if (end) {
 162         end->next = p;
 163     } else {
 164         head = p;
 165     }
 166 
 167     return head;
 168 }
 169 
 170 void
 171 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 172 {
 173     lrmd_key_value_t *p;
 174 
 175     while (head) {
 176         p = head->next;
 177         free(head->key);
 178         free(head->value);
 179         free(head);
 180         head = p;
 181     }
 182 }
 183 
 184 /*!
 185  * \brief Create a new lrmd_event_data_t object
 186  *
 187  * \param[in] rsc_id       ID of resource involved in event
 188  * \param[in] task         Action name
 189  * \param[in] interval_ms  Action interval
 190  *
 191  * \return Newly allocated and initialized lrmd_event_data_t
 192  * \note This functions asserts on memory errors, so the return value is
 193  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 194  *       result with lrmd_free_event().
 195  */
 196 lrmd_event_data_t *
 197 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 198 {
 199     lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 200 
 201     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 202     event->rsc_id = pcmk__str_copy(rsc_id);
 203     event->op_type = pcmk__str_copy(task);
 204     event->interval_ms = interval_ms;
 205     return event;
 206 }
 207 
 208 lrmd_event_data_t *
 209 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 210 {
 211     lrmd_event_data_t *copy = NULL;
 212 
 213     copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 214 
 215     copy->type = event->type;
 216 
 217     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 218     copy->rsc_id = pcmk__str_copy(event->rsc_id);
 219     copy->op_type = pcmk__str_copy(event->op_type);
 220     copy->user_data = pcmk__str_copy(event->user_data);
 221     copy->output = pcmk__str_copy(event->output);
 222     copy->remote_nodename = pcmk__str_copy(event->remote_nodename);
 223     copy->exit_reason = pcmk__str_copy(event->exit_reason);
 224 
 225     copy->call_id = event->call_id;
 226     copy->timeout = event->timeout;
 227     copy->interval_ms = event->interval_ms;
 228     copy->start_delay = event->start_delay;
 229     copy->rsc_deleted = event->rsc_deleted;
 230     copy->rc = event->rc;
 231     copy->op_status = event->op_status;
 232     copy->t_run = event->t_run;
 233     copy->t_rcchange = event->t_rcchange;
 234     copy->exec_time = event->exec_time;
 235     copy->queue_time = event->queue_time;
 236     copy->connection_rc = event->connection_rc;
 237     copy->params = pcmk__str_table_dup(event->params);
 238 
 239     return copy;
 240 }
 241 
 242 /*!
 243  * \brief Free an executor event
 244  *
 245  * \param[in,out]  Executor event object to free
 246  */
 247 void
 248 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 249 {
 250     if (event == NULL) {
 251         return;
 252     }
 253     // @TODO Why are these const char *?
 254     free((void *) event->rsc_id);
 255     free((void *) event->op_type);
 256     free((void *) event->user_data);
 257     free((void *) event->remote_nodename);
 258     lrmd__reset_result(event);
 259     if (event->params != NULL) {
 260         g_hash_table_destroy(event->params);
 261     }
 262     free(event);
 263 }
 264 
 265 static void
 266 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 267 {
 268     const char *type;
 269     const char *proxy_session = crm_element_value(msg,
 270                                                   PCMK__XA_LRMD_IPC_SESSION);
 271     lrmd_private_t *native = lrmd->lrmd_private;
 272     lrmd_event_data_t event = { 0, };
 273 
 274     if (proxy_session != NULL) {
 275         /* this is proxy business */
 276         lrmd_internal_proxy_dispatch(lrmd, msg);
 277         return;
 278     } else if (!native->callback) {
 279         /* no callback set */
 280         crm_trace("notify event received but client has not set callback");
 281         return;
 282     }
 283 
 284     event.remote_nodename = native->remote_nodename;
 285     type = crm_element_value(msg, PCMK__XA_LRMD_OP);
 286     crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
 287     event.rsc_id = crm_element_value(msg, PCMK__XA_LRMD_RSC_ID);
 288 
 289     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 290         event.type = lrmd_event_register;
 291     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 292         event.type = lrmd_event_unregister;
 293     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 294         int rc = 0;
 295         int exec_time = 0;
 296         int queue_time = 0;
 297         time_t epoch = 0;
 298 
 299         crm_element_value_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
 300         crm_element_value_ms(msg, PCMK__XA_LRMD_RSC_INTERVAL,
 301                              &event.interval_ms);
 302         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
 303                               &event.start_delay);
 304 
 305         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc);
 306         event.rc = (enum ocf_exitcode) rc;
 307 
 308         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS,
 309                               &event.op_status);
 310         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_DELETED,
 311                               &event.rsc_deleted);
 312 
 313         crm_element_value_epoch(msg, PCMK__XA_LRMD_RUN_TIME, &epoch);
 314         event.t_run = (unsigned int) epoch;
 315 
 316         crm_element_value_epoch(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &epoch);
 317         event.t_rcchange = (unsigned int) epoch;
 318 
 319         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time);
 320         CRM_LOG_ASSERT(exec_time >= 0);
 321         event.exec_time = QB_MAX(0, exec_time);
 322 
 323         crm_element_value_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time);
 324         CRM_LOG_ASSERT(queue_time >= 0);
 325         event.queue_time = QB_MAX(0, queue_time);
 326 
 327         event.op_type = crm_element_value(msg, PCMK__XA_LRMD_RSC_ACTION);
 328         event.user_data = crm_element_value(msg,
 329                                             PCMK__XA_LRMD_RSC_USERDATA_STR);
 330         event.type = lrmd_event_exec_complete;
 331 
 332         /* output and exit_reason may be freed by a callback */
 333         event.output = crm_element_value_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
 334         lrmd__set_result(&event, event.rc, event.op_status,
 335                          crm_element_value(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));
 336 
 337         event.params = xml2list(msg);
 338     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 339         event.type = lrmd_event_new_client;
 340     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 341         event.type = lrmd_event_poke;
 342     } else {
 343         return;
 344     }
 345 
 346     crm_trace("op %s notify event received", type);
 347     native->callback(&event);
 348 
 349     if (event.params) {
 350         g_hash_table_destroy(event.params);
 351     }
 352     lrmd__reset_result(&event);
 353 }
 354 
 355 // \return Always 0, to indicate that IPC mainloop source should be kept
 356 static int
 357 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 358 {
 359     lrmd_t *lrmd = userdata;
 360     lrmd_private_t *native = lrmd->lrmd_private;
 361 
 362     if (native->callback != NULL) {
 363         xmlNode *msg = pcmk__xml_parse(buffer);
 364 
 365         lrmd_dispatch_internal(lrmd, msg);
 366         free_xml(msg);
 367     }
 368     return 0;
 369 }
 370 
 371 #ifdef HAVE_GNUTLS_GNUTLS_H
 372 static void
 373 lrmd_free_xml(gpointer userdata)
     /*  */
 374 {
 375     free_xml((xmlNode *) userdata);
 376 }
 377 
 378 static bool
 379 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 380 {
 381     lrmd_private_t *native = lrmd->lrmd_private;
 382 
 383     return (native->remote->tls_session != NULL);
 384 }
 385 
 386 /*!
 387  * \internal
 388  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 389  *
 390  * \param[in,out] userdata  API connection
 391  *
 392  * \return Always return a nonnegative value, which as a file descriptor
 393  *         dispatch function means keep the mainloop source, and as a
 394  *         trigger dispatch function, 0 means remove the trigger from the
 395  *         mainloop while 1 means keep it (and job completed)
 396  */
 397 static int
 398 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 399 {
 400     lrmd_t *lrmd = userdata;
 401     lrmd_private_t *native = lrmd->lrmd_private;
 402     xmlNode *xml = NULL;
 403     int rc = pcmk_rc_ok;
 404 
 405     if (!remote_executor_connected(lrmd)) {
 406         crm_trace("TLS dispatch triggered after disconnect");
 407         return 0;
 408     }
 409 
 410     crm_trace("TLS dispatch triggered");
 411 
 412     /* First check if there are any pending notifies to process that came
 413      * while we were waiting for replies earlier. */
 414     if (native->pending_notify) {
 415         GList *iter = NULL;
 416 
 417         crm_trace("Processing pending notifies");
 418         for (iter = native->pending_notify; iter; iter = iter->next) {
 419             lrmd_dispatch_internal(lrmd, iter->data);
 420         }
 421         g_list_free_full(native->pending_notify, lrmd_free_xml);
 422         native->pending_notify = NULL;
 423     }
 424 
 425     /* Next read the current buffer and see if there are any messages to handle. */
 426     switch (pcmk__remote_ready(native->remote, 0)) {
 427         case pcmk_rc_ok:
 428             rc = pcmk__read_remote_message(native->remote, -1);
 429             xml = pcmk__remote_message_xml(native->remote);
 430             break;
 431         case ETIME:
 432             // Nothing to read, check if a full message is already in buffer
 433             xml = pcmk__remote_message_xml(native->remote);
 434             break;
 435         default:
 436             rc = ENOTCONN;
 437             break;
 438     }
 439     while (xml) {
 440         const char *msg_type = crm_element_value(xml,
 441                                                  PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 442         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 443             lrmd_dispatch_internal(lrmd, xml);
 444         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 445             if (native->expected_late_replies > 0) {
 446                 native->expected_late_replies--;
 447             } else {
 448                 int reply_id = 0;
 449                 crm_element_value_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
 450                 /* if this happens, we want to know about it */
 451                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 452             }
 453         }
 454         free_xml(xml);
 455         xml = pcmk__remote_message_xml(native->remote);
 456     }
 457 
 458     if (rc == ENOTCONN) {
 459         crm_info("Lost %s executor connection while reading data",
 460                  (native->remote_nodename? native->remote_nodename : "local"));
 461         lrmd_tls_disconnect(lrmd);
 462         return 0;
 463     }
 464     return 1;
 465 }
 466 #endif
 467 
 468 /* Not used with mainloop */
 469 int
 470 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 471 {
 472     lrmd_private_t *native = lrmd->lrmd_private;
 473 
 474     switch (native->type) {
 475         case pcmk__client_ipc:
 476             return crm_ipc_ready(native->ipc);
 477 
 478 #ifdef HAVE_GNUTLS_GNUTLS_H
 479         case pcmk__client_tls:
 480             if (native->pending_notify) {
 481                 return 1;
 482             } else {
 483                 int rc = pcmk__remote_ready(native->remote, 0);
 484 
 485                 switch (rc) {
 486                     case pcmk_rc_ok:
 487                         return 1;
 488                     case ETIME:
 489                         return 0;
 490                     default:
 491                         return pcmk_rc2legacy(rc);
 492                 }
 493             }
 494 #endif
 495         default:
 496             crm_err("Unsupported executor connection type (bug?): %d",
 497                     native->type);
 498             return -EPROTONOSUPPORT;
 499     }
 500 }
 501 
 502 /* Not used with mainloop */
 503 bool
 504 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 505 {
 506     lrmd_private_t *private = NULL;
 507 
 508     CRM_ASSERT(lrmd != NULL);
 509 
 510     private = lrmd->lrmd_private;
 511     switch (private->type) {
 512         case pcmk__client_ipc:
 513             while (crm_ipc_ready(private->ipc)) {
 514                 if (crm_ipc_read(private->ipc) > 0) {
 515                     const char *msg = crm_ipc_buffer(private->ipc);
 516 
 517                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 518                 }
 519             }
 520             break;
 521 #ifdef HAVE_GNUTLS_GNUTLS_H
 522         case pcmk__client_tls:
 523             lrmd_tls_dispatch(lrmd);
 524             break;
 525 #endif
 526         default:
 527             crm_err("Unsupported executor connection type (bug?): %d",
 528                     private->type);
 529     }
 530 
 531     if (lrmd_api_is_connected(lrmd) == FALSE) {
 532         crm_err("Connection closed");
 533         return FALSE;
 534     }
 535 
 536     return TRUE;
 537 }
 538 
 539 static xmlNode *
 540 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 541                enum lrmd_call_options options)
 542 {
 543     xmlNode *op_msg = NULL;
 544 
 545     CRM_CHECK(token != NULL, return NULL);
 546 
 547     op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
 548     crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
 549     crm_xml_add(op_msg, PCMK__XA_LRMD_OP, op);
 550     crm_xml_add_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
 551     crm_xml_add_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);
 552 
 553     if (data != NULL) {
 554         xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA);
 555 
 556         pcmk__xml_copy(wrapper, data);
 557     }
 558 
 559     crm_trace("Created executor %s command with call options %.8lx (%d)",
 560               op, (long)options, options);
 561     return op_msg;
 562 }
 563 
 564 static void
 565 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 566 {
 567     lrmd_t *lrmd = userdata;
 568     lrmd_private_t *native = lrmd->lrmd_private;
 569 
 570     switch (native->type) {
 571         case pcmk__client_ipc:
 572             crm_info("Disconnected from local executor");
 573             break;
 574 #ifdef HAVE_GNUTLS_GNUTLS_H
 575         case pcmk__client_tls:
 576             crm_info("Disconnected from remote executor on %s",
 577                      native->remote_nodename);
 578             break;
 579 #endif
 580         default:
 581             crm_err("Unsupported executor connection type %d (bug?)",
 582                     native->type);
 583     }
 584 
 585     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 586     native->ipc = NULL;
 587     native->source = NULL;
 588 
 589     if (native->callback) {
 590         lrmd_event_data_t event = { 0, };
 591         event.type = lrmd_event_disconnect;
 592         event.remote_nodename = native->remote_nodename;
 593         native->callback(&event);
 594     }
 595 }
 596 
 597 #ifdef HAVE_GNUTLS_GNUTLS_H
 598 static void
 599 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 600 {
 601     lrmd_t *lrmd = userdata;
 602     lrmd_private_t *native = lrmd->lrmd_private;
 603 
 604     crm_info("TLS connection destroyed");
 605 
 606     if (native->remote->tls_session) {
 607         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 608         gnutls_deinit(*native->remote->tls_session);
 609         gnutls_free(native->remote->tls_session);
 610         native->remote->tls_session = NULL;
 611     }
 612     if (native->psk_cred_c) {
 613         gnutls_psk_free_client_credentials(native->psk_cred_c);
 614     }
 615     if (native->sock) {
 616         close(native->sock);
 617     }
 618     if (native->process_notify) {
 619         mainloop_destroy_trigger(native->process_notify);
 620         native->process_notify = NULL;
 621     }
 622     if (native->pending_notify) {
 623         g_list_free_full(native->pending_notify, lrmd_free_xml);
 624         native->pending_notify = NULL;
 625     }
 626 
 627     free(native->remote->buffer);
 628     free(native->remote->start_state);
 629     native->remote->buffer = NULL;
 630     native->remote->start_state = NULL;
 631     native->source = 0;
 632     native->sock = 0;
 633     native->psk_cred_c = NULL;
 634     native->sock = 0;
 635 
 636     if (native->callback) {
 637         lrmd_event_data_t event = { 0, };
 638         event.remote_nodename = native->remote_nodename;
 639         event.type = lrmd_event_disconnect;
 640         native->callback(&event);
 641     }
 642     return;
 643 }
 644 
 645 // \return Standard Pacemaker return code
 646 int
 647 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 648                       const char *msg_type)
 649 {
 650     crm_xml_add_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
 651     crm_xml_add(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
 652     return pcmk__remote_send_xml(session, msg);
 653 }
 654 
 655 // \return Standard Pacemaker return code
 656 static int
 657 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 658                   xmlNode **reply)
 659 {
 660     lrmd_private_t *native = lrmd->lrmd_private;
 661     time_t start = time(NULL);
 662     const char *msg_type = NULL;
 663     int reply_id = 0;
 664     int remaining_timeout = 0;
 665     int rc = pcmk_rc_ok;
 666 
 667     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 668      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 669     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 670         total_timeout = MAX_TLS_RECV_WAIT;
 671     }
 672 
 673     for (*reply = NULL; *reply == NULL; ) {
 674 
 675         *reply = pcmk__remote_message_xml(native->remote);
 676         if (*reply == NULL) {
 677             /* read some more off the tls buffer if we still have time left. */
 678             if (remaining_timeout) {
 679                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 680             } else {
 681                 remaining_timeout = total_timeout;
 682             }
 683             if (remaining_timeout <= 0) {
 684                 return ETIME;
 685             }
 686 
 687             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 688             if (rc != pcmk_rc_ok) {
 689                 return rc;
 690             }
 691 
 692             *reply = pcmk__remote_message_xml(native->remote);
 693             if (*reply == NULL) {
 694                 return ENOMSG;
 695             }
 696         }
 697 
 698         crm_element_value_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
 699         msg_type = crm_element_value(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 700 
 701         if (!msg_type) {
 702             crm_err("Empty msg type received while waiting for reply");
 703             free_xml(*reply);
 704             *reply = NULL;
 705         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 706             /* got a notify while waiting for reply, trigger the notify to be processed later */
 707             crm_info("queueing notify");
 708             native->pending_notify = g_list_append(native->pending_notify, *reply);
 709             if (native->process_notify) {
 710                 crm_info("notify trigger set.");
 711                 mainloop_set_trigger(native->process_notify);
 712             }
 713             *reply = NULL;
 714         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 715             /* msg isn't a reply, make some noise */
 716             crm_err("Expected a reply, got %s", msg_type);
 717             free_xml(*reply);
 718             *reply = NULL;
 719         } else if (reply_id != expected_reply_id) {
 720             if (native->expected_late_replies > 0) {
 721                 native->expected_late_replies--;
 722             } else {
 723                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 724             }
 725             free_xml(*reply);
 726             *reply = NULL;
 727         }
 728     }
 729 
 730     if (native->remote->buffer && native->process_notify) {
 731         mainloop_set_trigger(native->process_notify);
 732     }
 733 
 734     return rc;
 735 }
 736 
 737 // \return Standard Pacemaker return code
 738 static int
 739 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 740 {
 741     int rc = pcmk_rc_ok;
 742     lrmd_private_t *native = lrmd->lrmd_private;
 743 
 744     global_remote_msg_id++;
 745     if (global_remote_msg_id <= 0) {
 746         global_remote_msg_id = 1;
 747     }
 748 
 749     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 750                                "request");
 751     if (rc != pcmk_rc_ok) {
 752         crm_err("Disconnecting because TLS message could not be sent to "
 753                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 754         lrmd_tls_disconnect(lrmd);
 755     }
 756     return rc;
 757 }
 758 
 759 static int
 760 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 761 {
 762     int rc = 0;
 763     xmlNode *xml = NULL;
 764 
 765     if (!remote_executor_connected(lrmd)) {
 766         return -ENOTCONN;
 767     }
 768 
 769     rc = send_remote_message(lrmd, msg);
 770     if (rc != pcmk_rc_ok) {
 771         return pcmk_rc2legacy(rc);
 772     }
 773 
 774     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 775     if (rc != pcmk_rc_ok) {
 776         crm_err("Disconnecting remote after request %d reply not received: %s "
 777                 CRM_XS " rc=%d timeout=%dms",
 778                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 779         lrmd_tls_disconnect(lrmd);
 780     }
 781 
 782     if (reply) {
 783         *reply = xml;
 784     } else {
 785         free_xml(xml);
 786     }
 787 
 788     return pcmk_rc2legacy(rc);
 789 }
 790 #endif
 791 
 792 static int
 793 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 794 {
 795     int rc = pcmk_ok;
 796     lrmd_private_t *native = lrmd->lrmd_private;
 797 
 798     switch (native->type) {
 799         case pcmk__client_ipc:
 800             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 801             break;
 802 #ifdef HAVE_GNUTLS_GNUTLS_H
 803         case pcmk__client_tls:
 804             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 805             break;
 806 #endif
 807         default:
 808             crm_err("Unsupported executor connection type (bug?): %d",
 809                     native->type);
 810             rc = -EPROTONOSUPPORT;
 811     }
 812 
 813     return rc;
 814 }
 815 
 816 static int
 817 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 818 {
 819     int rc = pcmk_ok;
 820     lrmd_private_t *native = lrmd->lrmd_private;
 821 
 822     switch (native->type) {
 823         case pcmk__client_ipc:
 824             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 825             break;
 826 #ifdef HAVE_GNUTLS_GNUTLS_H
 827         case pcmk__client_tls:
 828             rc = send_remote_message(lrmd, msg);
 829             if (rc == pcmk_rc_ok) {
 830                 /* we don't want to wait around for the reply, but
 831                  * since the request/reply protocol needs to behave the same
 832                  * as libqb, a reply will eventually come later anyway. */
 833                 native->expected_late_replies++;
 834             }
 835             rc = pcmk_rc2legacy(rc);
 836             break;
 837 #endif
 838         default:
 839             crm_err("Unsupported executor connection type (bug?): %d",
 840                     native->type);
 841             rc = -EPROTONOSUPPORT;
 842     }
 843 
 844     return rc;
 845 }
 846 
 847 static int
 848 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 849 {
 850     lrmd_private_t *native = lrmd->lrmd_private;
 851 
 852     switch (native->type) {
 853         case pcmk__client_ipc:
 854             return crm_ipc_connected(native->ipc);
 855 #ifdef HAVE_GNUTLS_GNUTLS_H
 856         case pcmk__client_tls:
 857             return remote_executor_connected(lrmd);
 858 #endif
 859         default:
 860             crm_err("Unsupported executor connection type (bug?): %d",
 861                     native->type);
 862             return 0;
 863     }
 864 }
 865 
 866 /*!
 867  * \internal
 868  * \brief Send a prepared API command to the executor
 869  *
 870  * \param[in,out] lrmd          Existing connection to the executor
 871  * \param[in]     op            Name of API command to send
 872  * \param[in]     data          Command data XML to add to the sent command
 873  * \param[out]    output_data   If expecting a reply, it will be stored here
 874  * \param[in]     timeout       Timeout in milliseconds (if 0, defaults to
 875  *                              a sensible value per the type of connection,
 876  *                              standard vs. pacemaker remote);
 877  *                              also propagated to the command XML
 878  * \param[in]     call_options  Call options to pass to server when sending
 879  * \param[in]     expect_reply  If TRUE, wait for a reply from the server;
 880  *                              must be TRUE for IPC (as opposed to TLS) clients
 881  *
 882  * \return pcmk_ok on success, -errno on error
 883  */
 884 static int
 885 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 886                   xmlNode **output_data, int timeout,
 887                   enum lrmd_call_options options, gboolean expect_reply)
 888 {
 889     int rc = pcmk_ok;
 890     lrmd_private_t *native = lrmd->lrmd_private;
 891     xmlNode *op_msg = NULL;
 892     xmlNode *op_reply = NULL;
 893 
 894     if (!lrmd_api_is_connected(lrmd)) {
 895         return -ENOTCONN;
 896     }
 897 
 898     if (op == NULL) {
 899         crm_err("No operation specified");
 900         return -EINVAL;
 901     }
 902 
 903     CRM_CHECK(native->token != NULL,;
 904         );
 905     crm_trace("Sending %s op to executor", op);
 906 
 907     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 908 
 909     if (op_msg == NULL) {
 910         return -EINVAL;
 911     }
 912 
 913     if (expect_reply) {
 914         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 915     } else {
 916         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 917         goto done;
 918     }
 919 
 920     if (rc < 0) {
 921         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 922         goto done;
 923 
 924     } else if(op_reply == NULL) {
 925         rc = -ENOMSG;
 926         goto done;
 927     }
 928 
 929     rc = pcmk_ok;
 930     crm_trace("%s op reply received", op);
 931     if (crm_element_value_int(op_reply, PCMK__XA_LRMD_RC, &rc) != 0) {
 932         rc = -ENOMSG;
 933         goto done;
 934     }
 935 
 936     crm_log_xml_trace(op_reply, "Reply");
 937 
 938     if (output_data) {
 939         *output_data = op_reply;
 940         op_reply = NULL;        /* Prevent subsequent free */
 941     }
 942 
 943   done:
 944     if (lrmd_api_is_connected(lrmd) == FALSE) {
 945         crm_err("Executor disconnected");
 946     }
 947 
 948     free_xml(op_msg);
 949     free_xml(op_reply);
 950     return rc;
 951 }
 952 
 953 static int
 954 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 955 {
 956     int rc;
 957     lrmd_private_t *native = lrmd->lrmd_private;
 958     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
 959 
 960     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
 961     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 962                            (native->type == pcmk__client_ipc));
 963     free_xml(data);
 964 
 965     return rc < 0 ? rc : pcmk_ok;
 966 }
 967 
 968 // \return Standard Pacemaker return code
 969 int
 970 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 971 {
 972     int rc = pcmk_rc_ok;
 973     const char *value;
 974     lrmd_private_t *native = lrmd->lrmd_private;
 975     xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP);
 976 
 977     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
 978 
 979     value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
 980     if ((value) &&
 981         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 982        crm_xml_add(data, PCMK__XA_LRMD_WATCHDOG, value);
 983     }
 984 
 985     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 986                            (native->type == pcmk__client_ipc));
 987     free_xml(data);
 988     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 989 }
 990 
 991 static int
 992 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 993 {
 994     int rc = pcmk_ok;
 995     lrmd_private_t *native = lrmd->lrmd_private;
 996     xmlNode *reply = NULL;
 997     xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
 998 
 999     crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
1000     crm_xml_add(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
1001     crm_xml_add(hello, PCMK__XA_LRMD_CLIENTNAME, name);
1002     crm_xml_add(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
1003 
1004     /* advertise that we are a proxy provider */
1005     if (native->proxy_callback) {
1006         pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
1007     }
1008 
1009     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
1010 
1011     if (rc < 0) {
1012         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
1013         rc = -ECOMM;
1014     } else if (reply == NULL) {
1015         crm_err("Did not receive registration reply");
1016         rc = -EPROTO;
1017     } else {
1018         const char *version = crm_element_value(reply,
1019                                                 PCMK__XA_LRMD_PROTOCOL_VERSION);
1020         const char *msg_type = crm_element_value(reply, PCMK__XA_LRMD_OP);
1021         const char *tmp_ticket = crm_element_value(reply,
1022                                                    PCMK__XA_LRMD_CLIENTID);
1023         const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
1024         long long uptime = -1;
1025 
1026         crm_element_value_int(reply, PCMK__XA_LRMD_RC, &rc);
1027 
1028         /* The remote executor may add its uptime to the XML reply, which is
1029          * useful in handling transient attributes when the connection to the
1030          * remote node unexpectedly drops.  If no parameter is given, just
1031          * default to -1.
1032          */
1033         crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
1034         native->remote->uptime = uptime;
1035 
1036         if (start_state) {
1037             native->remote->start_state = strdup(start_state);
1038         }
1039 
1040         if (rc == -EPROTO) {
1041             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
1042                 LRMD_PROTOCOL_VERSION, version);
1043             crm_log_xml_err(reply, "Protocol Error");
1044 
1045         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
1046             crm_err("Invalid registration message: %s", msg_type);
1047             crm_log_xml_err(reply, "Bad reply");
1048             rc = -EPROTO;
1049         } else if (tmp_ticket == NULL) {
1050             crm_err("No registration token provided");
1051             crm_log_xml_err(reply, "Bad reply");
1052             rc = -EPROTO;
1053         } else {
1054             crm_trace("Obtained registration token: %s", tmp_ticket);
1055             native->token = strdup(tmp_ticket);
1056             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1057             rc = pcmk_ok;
1058         }
1059     }
1060 
1061     free_xml(reply);
1062     free_xml(hello);
1063 
1064     if (rc != pcmk_ok) {
1065         lrmd_api_disconnect(lrmd);
1066     }
1067     return rc;
1068 }
1069 
1070 static int
1071 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1072 {
1073     int rc = pcmk_ok;
1074     lrmd_private_t *native = lrmd->lrmd_private;
1075 
1076     struct ipc_client_callbacks lrmd_callbacks = {
1077         .dispatch = lrmd_ipc_dispatch,
1078         .destroy = lrmd_ipc_connection_destroy
1079     };
1080 
1081     crm_info("Connecting to executor");
1082 
1083     if (fd) {
1084         /* No mainloop */
1085         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1086         if (native->ipc != NULL) {
1087             rc = pcmk__connect_generic_ipc(native->ipc);
1088             if (rc == pcmk_rc_ok) {
1089                 rc = pcmk__ipc_fd(native->ipc, fd);
1090             }
1091             if (rc != pcmk_rc_ok) {
1092                 crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
1093                 rc = -ENOTCONN;
1094             }
1095         }
1096     } else {
1097         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1098         native->ipc = mainloop_get_ipc_client(native->source);
1099     }
1100 
1101     if (native->ipc == NULL) {
1102         crm_debug("Could not connect to the executor API");
1103         rc = -ENOTCONN;
1104     }
1105 
1106     return rc;
1107 }
1108 
1109 #ifdef HAVE_GNUTLS_GNUTLS_H
1110 static void
1111 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1112 {
1113     CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
1114 
1115     dest->data = gnutls_malloc(source->size);
1116     pcmk__mem_assert(dest->data);
1117 
1118     memcpy(dest->data, source->data, source->size);
1119     dest->size = source->size;
1120 }
1121 
1122 static void
1123 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1124 {
1125     gnutls_free(datum->data);
1126     datum->data = NULL;
1127     datum->size = 0;
1128 }
1129 
1130 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1131 
1132 // \return Standard Pacemaker return code
1133 static int
1134 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1135 {
1136     FILE *stream = NULL;
1137     size_t buf_len = KEY_READ_LEN;
1138 
1139     if ((location == NULL) || (key == NULL)) {
1140         return EINVAL;
1141     }
1142 
1143     stream = fopen(location, "r");
1144     if (stream == NULL) {
1145         return errno;
1146     }
1147 
1148     key->data = gnutls_malloc(buf_len);
1149     key->size = 0;
1150     while (!feof(stream)) {
1151         int next = fgetc(stream);
1152 
1153         if (next == EOF) {
1154             if (!feof(stream)) {
1155                 crm_warn("Pacemaker Remote key read was partially successful "
1156                          "(copy in memory may be corrupted)");
1157             }
1158             break;
1159         }
1160         if (key->size == buf_len) {
1161             buf_len = key->size + KEY_READ_LEN;
1162             key->data = gnutls_realloc(key->data, buf_len);
1163             CRM_ASSERT(key->data);
1164         }
1165         key->data[key->size++] = (unsigned char) next;
1166     }
1167     fclose(stream);
1168 
1169     if (key->size == 0) {
1170         clear_gnutls_datum(key);
1171         return ENOKEY;
1172     }
1173     return pcmk_rc_ok;
1174 }
1175 
1176 // Cache the most recently used Pacemaker Remote authentication key
1177 
1178 struct key_cache_s {
1179     time_t updated;         // When cached key was read (valid for 1 minute)
1180     const char *location;   // Where cached key was read from
1181     gnutls_datum_t key;     // Cached key
1182 };
1183 
1184 static bool
1185 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1186 {
1187     return key_cache->updated != 0;
1188 }
1189 
1190 static bool
1191 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1192 {
1193     return (time(NULL) - key_cache->updated) >= 60;
1194 }
1195 
1196 static void
1197 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1198 {
1199     clear_gnutls_datum(&(key_cache->key));
1200     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1201         key_cache->updated = 0;
1202         key_cache->location = NULL;
1203         crm_debug("Cleared Pacemaker Remote key cache");
1204     }
1205 }
1206 
1207 static void
1208 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1209 {
1210     copy_gnutls_datum(key, &(key_cache->key));
1211     crm_debug("Using cached Pacemaker Remote key from %s",
1212               pcmk__s(key_cache->location, "unknown location"));
1213 }
1214 
1215 static void
1216 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1217           const char *location)
1218 {
1219     key_cache->updated = time(NULL);
1220     key_cache->location = location;
1221     copy_gnutls_datum(&(key_cache->key), key);
1222     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1223               pcmk__s(location, "unknown location"));
1224 }
1225 
1226 /*!
1227  * \internal
1228  * \brief Get Pacemaker Remote authentication key from file or cache
1229  *
1230  * \param[in]  location         Path to key file to try (this memory must
1231  *                              persist across all calls of this function)
1232  * \param[out] key              Key from location or cache
1233  *
1234  * \return Standard Pacemaker return code
1235  */
1236 static int
1237 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1238 {
1239     static struct key_cache_s key_cache = { 0, };
1240     int rc = pcmk_rc_ok;
1241 
1242     if ((location == NULL) || (key == NULL)) {
1243         return EINVAL;
1244     }
1245 
1246     if (key_is_cached(&key_cache)) {
1247         if (key_cache_expired(&key_cache)) {
1248             clear_key_cache(&key_cache);
1249         } else {
1250             get_cached_key(&key_cache, key);
1251             return pcmk_rc_ok;
1252         }
1253     }
1254 
1255     rc = read_gnutls_key(location, key);
1256     if (rc != pcmk_rc_ok) {
1257         return rc;
1258     }
1259     cache_key(&key_cache, key, location);
1260     return pcmk_rc_ok;
1261 }
1262 
1263 /*!
1264  * \internal
1265  * \brief Initialize the Pacemaker Remote authentication key
1266  *
1267  * Try loading the Pacemaker Remote authentication key from cache if available,
1268  * otherwise from these locations, in order of preference: the value of the
1269  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1270  * file location; or (for historical reasons) /etc/corosync/authkey.
1271  *
1272  * \param[out] key  Where to store key
1273  *
1274  * \return Standard Pacemaker return code
1275  */
1276 int
1277 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1278 {
1279     static const char *env_location = NULL;
1280     static bool need_env = true;
1281 
1282     int env_rc = pcmk_rc_ok;
1283     int default_rc = pcmk_rc_ok;
1284     int alt_rc = pcmk_rc_ok;
1285 
1286     bool env_is_default = false;
1287     bool env_is_fallback = false;
1288 
1289     if (need_env) {
1290         env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
1291         need_env = false;
1292     }
1293 
1294     // Try location in environment variable, if set
1295     if (env_location != NULL) {
1296         env_rc = get_remote_key(env_location, key);
1297         if (env_rc == pcmk_rc_ok) {
1298             return pcmk_rc_ok;
1299         }
1300 
1301         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1302         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1303 
1304         /* @TODO It would be more secure to fail, rather than fall back to the
1305          * default, if an explicitly set key location is not readable, and it
1306          * would be better to never use the Corosync location as a fallback.
1307          * However, that would break any deployments currently working with the
1308          * fallbacks.
1309          *
1310          * @COMPAT Change at 3.0.0
1311          */
1312     }
1313 
1314     // Try default location, if environment wasn't explicitly set to it
1315     if (env_is_default) {
1316         default_rc = env_rc;
1317     } else {
1318         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1319     }
1320 
1321     // Try fallback location, if environment wasn't set to it and default failed
1322     // @COMPAT Drop at 3.0.0
1323     if (env_is_fallback) {
1324         alt_rc = env_rc;
1325     } else if (default_rc != pcmk_rc_ok) {
1326         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1327     }
1328 
1329     // We have all results, so log and return
1330 
1331     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1332         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1333 
1334         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1335                  env_location,
1336                  env_is_default? "" : "or default location ",
1337                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1338                  !env_is_default && !env_is_fallback? " " : "",
1339                  env_is_fallback? "" : "or fallback location ",
1340                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1341                  pcmk_rc_str(env_rc));
1342         return ENOKEY;
1343     }
1344 
1345     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1346         crm_warn("Could not read Pacemaker Remote key from %s "
1347                  "(using %s location %s instead): %s",
1348                  env_location,
1349                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1350                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1351                  pcmk_rc_str(env_rc));
1352         crm_warn("This undocumented behavior is deprecated and unsafe and will "
1353                  "be removed in a future release");
1354         return pcmk_rc_ok;
1355     }
1356 
1357     if (default_rc != pcmk_rc_ok) {
1358         if (alt_rc == pcmk_rc_ok) {
1359             // Environment variable unset, used alternate location
1360             // This gets caught by the default return below, but we additionally
1361             // warn on this behavior here.
1362             crm_warn("Read Pacemaker Remote key from alternate location %s",
1363                      ALT_REMOTE_KEY_LOCATION);
1364             crm_warn("This undocumented behavior is deprecated and unsafe and will "
1365                      "be removed in a future release");
1366         } else {
1367             // Environment unset, defaults failed
1368             crm_warn("Could not read Pacemaker Remote key from default location %s"
1369                      " (or fallback location %s): %s",
1370                      DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1371                      pcmk_rc_str(default_rc));
1372             return ENOKEY;
1373         }
1374     }
1375 
1376     return pcmk_rc_ok; // Environment variable unset, a default worked
1377 }
1378 
1379 static void
1380 lrmd_gnutls_global_init(void)
     /*  */
1381 {
1382     static int gnutls_init = 0;
1383 
1384     if (!gnutls_init) {
1385         crm_gnutls_global_init();
1386     }
1387     gnutls_init = 1;
1388 }
1389 #endif
1390 
1391 static void
1392 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1393 {
1394     lrmd_private_t *native = lrmd->lrmd_private;
1395 
1396     if (native->callback) {
1397         lrmd_event_data_t event = { 0, };
1398         event.type = lrmd_event_connect;
1399         event.remote_nodename = native->remote_nodename;
1400         event.connection_rc = rc;
1401         native->callback(&event);
1402     }
1403 }
1404 
1405 #ifdef HAVE_GNUTLS_GNUTLS_H
1406 /*!
1407  * \internal
1408  * \brief Perform a TLS client handshake with a Pacemaker Remote server
1409  *
1410  * \param[in] lrmd  Newly established Pacemaker Remote executor connection
1411  *
1412  * \return Standard Pacemaker return code
1413  */
1414 static int
1415 tls_client_handshake(lrmd_t *lrmd)
     /*  */
1416 {
1417     lrmd_private_t *native = lrmd->lrmd_private;
1418     int tls_rc = GNUTLS_E_SUCCESS;
1419     int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT,
1420                                         &tls_rc);
1421 
1422     if (rc != pcmk_rc_ok) {
1423         crm_warn("Disconnecting after TLS handshake with "
1424                  "Pacemaker Remote server %s:%d failed: %s",
1425                  native->server, native->port,
1426                  (rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc));
1427         gnutls_deinit(*native->remote->tls_session);
1428         gnutls_free(native->remote->tls_session);
1429         native->remote->tls_session = NULL;
1430         lrmd_tls_connection_destroy(lrmd);
1431     }
1432     return rc;
1433 }
1434 
1435 /*!
1436  * \internal
1437  * \brief Add trigger and file descriptor mainloop sources for TLS
1438  *
1439  * \param[in,out] lrmd          API connection with established TLS session
1440  * \param[in]     do_handshake  Whether to perform executor handshake
1441  *
1442  * \return Standard Pacemaker return code
1443  */
1444 static int
1445 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1446 {
1447     lrmd_private_t *native = lrmd->lrmd_private;
1448     int rc = pcmk_rc_ok;
1449 
1450     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1451                                    native->server, native->port);
1452 
1453     struct mainloop_fd_callbacks tls_fd_callbacks = {
1454         .dispatch = lrmd_tls_dispatch,
1455         .destroy = lrmd_tls_connection_destroy,
1456     };
1457 
1458     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1459                                                   lrmd_tls_dispatch, lrmd);
1460     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1461                                      &tls_fd_callbacks);
1462 
1463     /* Async connections lose the client name provided by the API caller, so we
1464      * have to use our generated name here to perform the executor handshake.
1465      *
1466      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1467      * that name in this function instead of generating one anyway.
1468      */
1469     if (do_handshake) {
1470         rc = lrmd_handshake(lrmd, name);
1471         rc = pcmk_legacy2rc(rc);
1472     }
1473     free(name);
1474     return rc;
1475 }
1476 
1477 static void
1478 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1479 {
1480     lrmd_t *lrmd = userdata;
1481     lrmd_private_t *native = lrmd->lrmd_private;
1482     gnutls_datum_t psk_key = { NULL, 0 };
1483 
1484     native->async_timer = 0;
1485 
1486     if (rc != pcmk_rc_ok) {
1487         lrmd_tls_connection_destroy(lrmd);
1488         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1489                  CRM_XS " rc=%d",
1490                  native->server, native->port, pcmk_rc_str(rc), rc);
1491         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1492         return;
1493     }
1494 
1495     /* The TCP connection was successful, so establish the TLS connection.
1496      * @TODO make this async to avoid blocking code in client
1497      */
1498 
1499     native->sock = sock;
1500 
1501     rc = lrmd__init_remote_key(&psk_key);
1502     if (rc != pcmk_rc_ok) {
1503         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1504                  CRM_XS " rc=%d",
1505                  native->server, native->port, pcmk_rc_str(rc), rc);
1506         lrmd_tls_connection_destroy(lrmd);
1507         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1508         return;
1509     }
1510 
1511     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1512     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1513     gnutls_free(psk_key.data);
1514 
1515     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1516                                                         GNUTLS_CRD_PSK,
1517                                                         native->psk_cred_c);
1518     if (native->remote->tls_session == NULL) {
1519         lrmd_tls_connection_destroy(lrmd);
1520         report_async_connection_result(lrmd, -EPROTO);
1521         return;
1522     }
1523 
1524     if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
1525         report_async_connection_result(lrmd, -EKEYREJECTED);
1526         return;
1527     }
1528 
1529     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1530              native->server, native->port);
1531     rc = add_tls_to_mainloop(lrmd, true);
1532     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1533 }
1534 
1535 static int
1536 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1537 {
1538     int rc;
1539     int timer_id = 0;
1540     lrmd_private_t *native = lrmd->lrmd_private;
1541 
1542     lrmd_gnutls_global_init();
1543     native->sock = -1;
1544     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1545                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1546     if (rc != pcmk_rc_ok) {
1547         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1548                  CRM_XS " rc=%d",
1549                  native->server, native->port, pcmk_rc_str(rc), rc);
1550         return pcmk_rc2legacy(rc);
1551     }
1552     native->async_timer = timer_id;
1553     return pcmk_ok;
1554 }
1555 
1556 static int
1557 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1558 {
1559     int rc;
1560 
1561     lrmd_private_t *native = lrmd->lrmd_private;
1562     gnutls_datum_t psk_key = { NULL, 0 };
1563 
1564     lrmd_gnutls_global_init();
1565 
1566     native->sock = -1;
1567     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1568                               &(native->sock), NULL, NULL);
1569     if (rc != pcmk_rc_ok) {
1570         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1571                  CRM_XS " rc=%d",
1572                  native->server, native->port, pcmk_rc_str(rc), rc);
1573         lrmd_tls_connection_destroy(lrmd);
1574         return -ENOTCONN;
1575     }
1576 
1577     rc = lrmd__init_remote_key(&psk_key);
1578     if (rc != pcmk_rc_ok) {
1579         lrmd_tls_connection_destroy(lrmd);
1580         return pcmk_rc2legacy(rc);
1581     }
1582 
1583     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1584     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1585     gnutls_free(psk_key.data);
1586 
1587     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1588                                                         GNUTLS_CRD_PSK,
1589                                                         native->psk_cred_c);
1590     if (native->remote->tls_session == NULL) {
1591         lrmd_tls_connection_destroy(lrmd);
1592         return -EPROTO;
1593     }
1594 
1595     if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
1596         return -EKEYREJECTED;
1597     }
1598 
1599     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1600              native->port);
1601 
1602     if (fd) {
1603         *fd = native->sock;
1604     } else {
1605         add_tls_to_mainloop(lrmd, false);
1606     }
1607     return pcmk_ok;
1608 }
1609 #endif
1610 
1611 static int
1612 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1613 {
1614     int rc = -ENOTCONN;
1615     lrmd_private_t *native = lrmd->lrmd_private;
1616 
1617     switch (native->type) {
1618         case pcmk__client_ipc:
1619             rc = lrmd_ipc_connect(lrmd, fd);
1620             break;
1621 #ifdef HAVE_GNUTLS_GNUTLS_H
1622         case pcmk__client_tls:
1623             rc = lrmd_tls_connect(lrmd, fd);
1624             break;
1625 #endif
1626         default:
1627             crm_err("Unsupported executor connection type (bug?): %d",
1628                     native->type);
1629             rc = -EPROTONOSUPPORT;
1630     }
1631 
1632     if (rc == pcmk_ok) {
1633         rc = lrmd_handshake(lrmd, name);
1634     }
1635 
1636     return rc;
1637 }
1638 
1639 static int
1640 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1641 {
1642     int rc = pcmk_ok;
1643     lrmd_private_t *native = lrmd->lrmd_private;
1644 
1645     CRM_CHECK(native && native->callback, return -EINVAL);
1646 
1647     switch (native->type) {
1648         case pcmk__client_ipc:
1649             /* fake async connection with ipc.  it should be fast
1650              * enough that we gain very little from async */
1651             rc = lrmd_api_connect(lrmd, name, NULL);
1652             if (!rc) {
1653                 report_async_connection_result(lrmd, rc);
1654             }
1655             break;
1656 #ifdef HAVE_GNUTLS_GNUTLS_H
1657         case pcmk__client_tls:
1658             rc = lrmd_tls_connect_async(lrmd, timeout);
1659             if (rc) {
1660                 /* connection failed, report rc now */
1661                 report_async_connection_result(lrmd, rc);
1662             }
1663             break;
1664 #endif
1665         default:
1666             crm_err("Unsupported executor connection type (bug?): %d",
1667                     native->type);
1668             rc = -EPROTONOSUPPORT;
1669     }
1670 
1671     return rc;
1672 }
1673 
1674 static void
1675 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1676 {
1677     lrmd_private_t *native = lrmd->lrmd_private;
1678 
1679     if (native->source != NULL) {
1680         /* Attached to mainloop */
1681         mainloop_del_ipc_client(native->source);
1682         native->source = NULL;
1683         native->ipc = NULL;
1684 
1685     } else if (native->ipc) {
1686         /* Not attached to mainloop */
1687         crm_ipc_t *ipc = native->ipc;
1688 
1689         native->ipc = NULL;
1690         crm_ipc_close(ipc);
1691         crm_ipc_destroy(ipc);
1692     }
1693 }
1694 
1695 #ifdef HAVE_GNUTLS_GNUTLS_H
1696 static void
1697 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1698 {
1699     lrmd_private_t *native = lrmd->lrmd_private;
1700 
1701     if (native->remote->tls_session) {
1702         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1703         gnutls_deinit(*native->remote->tls_session);
1704         gnutls_free(native->remote->tls_session);
1705         native->remote->tls_session = NULL;
1706     }
1707 
1708     if (native->async_timer) {
1709         g_source_remove(native->async_timer);
1710         native->async_timer = 0;
1711     }
1712 
1713     if (native->source != NULL) {
1714         /* Attached to mainloop */
1715         mainloop_del_ipc_client(native->source);
1716         native->source = NULL;
1717 
1718     } else if (native->sock) {
1719         close(native->sock);
1720         native->sock = 0;
1721     }
1722 
1723     if (native->pending_notify) {
1724         g_list_free_full(native->pending_notify, lrmd_free_xml);
1725         native->pending_notify = NULL;
1726     }
1727 }
1728 #endif
1729 
1730 static int
1731 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1732 {
1733     lrmd_private_t *native = lrmd->lrmd_private;
1734     int rc = pcmk_ok;
1735 
1736     switch (native->type) {
1737         case pcmk__client_ipc:
1738             crm_debug("Disconnecting from local executor");
1739             lrmd_ipc_disconnect(lrmd);
1740             break;
1741 #ifdef HAVE_GNUTLS_GNUTLS_H
1742         case pcmk__client_tls:
1743             crm_debug("Disconnecting from remote executor on %s",
1744                       native->remote_nodename);
1745             lrmd_tls_disconnect(lrmd);
1746             break;
1747 #endif
1748         default:
1749             crm_err("Unsupported executor connection type (bug?): %d",
1750                     native->type);
1751             rc = -EPROTONOSUPPORT;
1752     }
1753 
1754     free(native->token);
1755     native->token = NULL;
1756 
1757     free(native->peer_version);
1758     native->peer_version = NULL;
1759     return rc;
1760 }
1761 
1762 static int
1763 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1764                       const char *rsc_id,
1765                       const char *class,
1766                       const char *provider, const char *type, enum lrmd_call_options options)
1767 {
1768     int rc = pcmk_ok;
1769     xmlNode *data = NULL;
1770 
1771     if (!class || !type || !rsc_id) {
1772         return -EINVAL;
1773     }
1774     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1775         && (provider == NULL)) {
1776         return -EINVAL;
1777     }
1778 
1779     data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1780 
1781     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1782     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1783     crm_xml_add(data, PCMK__XA_LRMD_CLASS, class);
1784     crm_xml_add(data, PCMK__XA_LRMD_PROVIDER, provider);
1785     crm_xml_add(data, PCMK__XA_LRMD_TYPE, type);
1786     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1787     free_xml(data);
1788 
1789     return rc;
1790 }
1791 
1792 static int
1793 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1794 {
1795     int rc = pcmk_ok;
1796     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1797 
1798     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1799     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1800     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1801     free_xml(data);
1802 
1803     return rc;
1804 }
1805 
1806 lrmd_rsc_info_t *
1807 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1808                   const char *provider, const char *type)
1809 {
1810     lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t));
1811 
1812     rsc_info->id = pcmk__str_copy(rsc_id);
1813     rsc_info->standard = pcmk__str_copy(standard);
1814     rsc_info->provider = pcmk__str_copy(provider);
1815     rsc_info->type = pcmk__str_copy(type);
1816     return rsc_info;
1817 }
1818 
1819 lrmd_rsc_info_t *
1820 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1821 {
1822     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1823                              rsc_info->provider, rsc_info->type);
1824 }
1825 
1826 void
1827 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1828 {
1829     if (!rsc_info) {
1830         return;
1831     }
1832     free(rsc_info->id);
1833     free(rsc_info->type);
1834     free(rsc_info->standard);
1835     free(rsc_info->provider);
1836     free(rsc_info);
1837 }
1838 
1839 static lrmd_rsc_info_t *
1840 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1841 {
1842     lrmd_rsc_info_t *rsc_info = NULL;
1843     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1844     xmlNode *output = NULL;
1845     const char *class = NULL;
1846     const char *provider = NULL;
1847     const char *type = NULL;
1848 
1849     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1850     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1851     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1852     free_xml(data);
1853 
1854     if (!output) {
1855         return NULL;
1856     }
1857 
1858     class = crm_element_value(output, PCMK__XA_LRMD_CLASS);
1859     provider = crm_element_value(output, PCMK__XA_LRMD_PROVIDER);
1860     type = crm_element_value(output, PCMK__XA_LRMD_TYPE);
1861 
1862     if (!class || !type) {
1863         free_xml(output);
1864         return NULL;
1865     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1866                && !provider) {
1867         free_xml(output);
1868         return NULL;
1869     }
1870 
1871     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1872     free_xml(output);
1873     return rsc_info;
1874 }
1875 
1876 void
1877 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1878 {
1879     if (op_info) {
1880         free(op_info->rsc_id);
1881         free(op_info->action);
1882         free(op_info->interval_ms_s);
1883         free(op_info->timeout_ms_s);
1884         free(op_info);
1885     }
1886 }
1887 
1888 static int
1889 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1890                            enum lrmd_call_options options, GList **output)
1891 {
1892     xmlNode *data = NULL;
1893     xmlNode *output_xml = NULL;
1894     int rc = pcmk_ok;
1895 
1896     if (output == NULL) {
1897         return -EINVAL;
1898     }
1899     *output = NULL;
1900 
1901     // Send request
1902     if (rsc_id) {
1903         data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1904         crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1905         crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1906     }
1907     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1908                            timeout_ms, options, TRUE);
1909     if (data) {
1910         free_xml(data);
1911     }
1912 
1913     // Process reply
1914     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1915         return rc;
1916     }
1917     for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml,
1918                                                        PCMK__XE_LRMD_RSC, NULL,
1919                                                        NULL);
1920          (rsc_xml != NULL) && (rc == pcmk_ok);
1921          rsc_xml = pcmk__xe_next_same(rsc_xml)) {
1922 
1923         rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
1924         if (rsc_id == NULL) {
1925             crm_err("Could not parse recurring operation information from executor");
1926             continue;
1927         }
1928         for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml,
1929                                                           PCMK__XE_LRMD_RSC_OP,
1930                                                           NULL, NULL);
1931              op_xml != NULL; op_xml = pcmk__xe_next_same(op_xml)) {
1932 
1933             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1934 
1935             if (op_info == NULL) {
1936                 rc = -ENOMEM;
1937                 break;
1938             }
1939             op_info->rsc_id = strdup(rsc_id);
1940             op_info->action = crm_element_value_copy(op_xml,
1941                                                      PCMK__XA_LRMD_RSC_ACTION);
1942             op_info->interval_ms_s =
1943                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
1944             op_info->timeout_ms_s =
1945                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_TIMEOUT);
1946             *output = g_list_prepend(*output, op_info);
1947         }
1948     }
1949     free_xml(output_xml);
1950     return rc;
1951 }
1952 
1953 
1954 static void
1955 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1956 {
1957     lrmd_private_t *native = lrmd->lrmd_private;
1958 
1959     native->callback = callback;
1960 }
1961 
1962 void
1963 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1964 {
1965     lrmd_private_t *native = lrmd->lrmd_private;
1966 
1967     native->proxy_callback = callback;
1968     native->proxy_callback_userdata = userdata;
1969 }
1970 
1971 void
1972 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1973 {
1974     lrmd_private_t *native = lrmd->lrmd_private;
1975 
1976     if (native->proxy_callback) {
1977         crm_log_xml_trace(msg, "PROXY_INBOUND");
1978         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1979     }
1980 }
1981 
1982 int
1983 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1984 {
1985     if (lrmd == NULL) {
1986         return -ENOTCONN;
1987     }
1988     crm_xml_add(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);
1989 
1990     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1991     return lrmd_send_xml_no_reply(lrmd, msg);
1992 }
1993 
1994 static int
1995 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1996 {
1997     int rc = pcmk_ok;
1998     stonith_t *stonith_api = stonith_api_new();
1999 
2000     if (stonith_api == NULL) {
2001         crm_err("Could not get fence agent meta-data: API memory allocation failed");
2002         return -ENOMEM;
2003     }
2004 
2005     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
2006                                      provider, output, 0);
2007     if ((rc == pcmk_ok) && (*output == NULL)) {
2008         rc = -EIO;
2009     }
2010     stonith_api->cmds->free(stonith_api);
2011     return rc;
2012 }
2013 
2014 static int
2015 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
2016                       const char *type, char **output,
2017                       enum lrmd_call_options options)
2018 {
2019     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
2020                                            output, options, NULL);
2021 }
2022 
2023 static int
2024 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
2025                              const char *provider, const char *type,
2026                              char **output, enum lrmd_call_options options,
2027                              lrmd_key_value_t *params)
2028 {
2029     svc_action_t *action = NULL;
2030     GHashTable *params_table = NULL;
2031 
2032     if (!standard || !type) {
2033         lrmd_key_value_freeall(params);
2034         return -EINVAL;
2035     }
2036 
2037     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2038         lrmd_key_value_freeall(params);
2039         return stonith_get_metadata(provider, type, output);
2040     }
2041 
2042     params_table = pcmk__strkey_table(free, free);
2043     for (const lrmd_key_value_t *param = params; param; param = param->next) {
2044         pcmk__insert_dup(params_table, param->key, param->value);
2045     }
2046     action = services__create_resource_action(type, standard, provider, type,
2047                                               PCMK_ACTION_META_DATA, 0,
2048                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
2049                                               params_table, 0);
2050     lrmd_key_value_freeall(params);
2051 
2052     if (action == NULL) {
2053         return -ENOMEM;
2054     }
2055     if (action->rc != PCMK_OCF_UNKNOWN) {
2056         services_action_free(action);
2057         return -EINVAL;
2058     }
2059 
2060     if (!services_action_sync(action)) {
2061         crm_err("Failed to retrieve meta-data for %s:%s:%s",
2062                 standard, provider, type);
2063         services_action_free(action);
2064         return -EIO;
2065     }
2066 
2067     if (!action->stdout_data) {
2068         crm_err("Failed to receive meta-data for %s:%s:%s",
2069                 standard, provider, type);
2070         services_action_free(action);
2071         return -EIO;
2072     }
2073 
2074     *output = strdup(action->stdout_data);
2075     services_action_free(action);
2076 
2077     return pcmk_ok;
2078 }
2079 
2080 static int
2081 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2082               const char *userdata, guint interval_ms,
2083               int timeout,      /* ms */
2084               int start_delay,  /* ms */
2085               enum lrmd_call_options options, lrmd_key_value_t * params)
2086 {
2087     int rc = pcmk_ok;
2088     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2089     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2090     lrmd_key_value_t *tmp = NULL;
2091 
2092     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2093     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2094     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2095     crm_xml_add(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
2096     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2097     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2098     crm_xml_add_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);
2099 
2100     for (tmp = params; tmp; tmp = tmp->next) {
2101         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2102     }
2103 
2104     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2105     free_xml(data);
2106 
2107     lrmd_key_value_freeall(params);
2108     return rc;
2109 }
2110 
2111 /* timeout is in ms */
2112 static int
2113 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2114                     int timeout, lrmd_key_value_t *params)
2115 {
2116     int rc = pcmk_ok;
2117     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT);
2118     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2119     lrmd_key_value_t *tmp = NULL;
2120 
2121     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2122     crm_xml_add(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
2123     crm_xml_add(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
2124     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2125 
2126     for (tmp = params; tmp; tmp = tmp->next) {
2127         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2128     }
2129 
2130     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2131                            lrmd_opt_notify_orig_only, TRUE);
2132     free_xml(data);
2133 
2134     lrmd_key_value_freeall(params);
2135     return rc;
2136 }
2137 
2138 static int
2139 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2140                 guint interval_ms)
2141 {
2142     int rc = pcmk_ok;
2143     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2144 
2145     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2146     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2147     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2148     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2149     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2150     free_xml(data);
2151     return rc;
2152 }
2153 
2154 static int
2155 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2156 {
2157     int rc = 0;
2158     stonith_t *stonith_api = stonith_api_new();
2159     stonith_key_value_t *stonith_resources = NULL;
2160     stonith_key_value_t *dIter = NULL;
2161 
2162     if (stonith_api == NULL) {
2163         crm_err("Could not list fence agents: API memory allocation failed");
2164         return -ENOMEM;
2165     }
2166     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2167                                    &stonith_resources, 0);
2168     stonith_api->cmds->free(stonith_api);
2169 
2170     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2171         rc++;
2172         if (resources) {
2173             *resources = lrmd_list_add(*resources, dIter->value);
2174         }
2175     }
2176 
2177     stonith_key_value_freeall(stonith_resources, 1, 0);
2178     return rc;
2179 }
2180 
2181 static int
2182 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2183                      const char *provider)
2184 {
2185     int rc = 0;
2186     int stonith_count = 0; // Initially, whether to include stonith devices
2187 
2188     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2189         stonith_count = 1;
2190 
2191     } else {
2192         GList *gIter = NULL;
2193         GList *agents = resources_list_agents(class, provider);
2194 
2195         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2196             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2197             rc++;
2198         }
2199         g_list_free_full(agents, free);
2200 
2201         if (!class) {
2202             stonith_count = 1;
2203         }
2204     }
2205 
2206     if (stonith_count) {
2207         // Now, if stonith devices are included, how many there are
2208         stonith_count = list_stonith_agents(resources);
2209         if (stonith_count > 0) {
2210             rc += stonith_count;
2211         }
2212     }
2213     if (rc == 0) {
2214         crm_notice("No agents found for class %s", class);
2215         rc = -EPROTONOSUPPORT;
2216     }
2217     return rc;
2218 }
2219 
2220 static bool
2221 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2222 {
2223     bool found = false;
2224     GList *agents = NULL;
2225     GList *gIter2 = NULL;
2226 
2227     agents = resources_list_agents(class, provider);
2228     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2229         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2230             found = true;
2231         }
2232     }
2233     g_list_free_full(agents, free);
2234     return found;
2235 }
2236 
2237 static int
2238 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2239 {
2240     int rc = pcmk_ok;
2241     char *provider = NULL;
2242     GList *ocf_providers = NULL;
2243     GList *gIter = NULL;
2244 
2245     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2246 
2247     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2248         provider = gIter->data;
2249         if (!agent || does_provider_have_agent(agent, provider,
2250                                                PCMK_RESOURCE_CLASS_OCF)) {
2251             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2252             rc++;
2253         }
2254     }
2255 
2256     g_list_free_full(ocf_providers, free);
2257     return rc;
2258 }
2259 
2260 static int
2261 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2262 {
2263     int rc = 0;
2264     GList *standards = NULL;
2265     GList *gIter = NULL;
2266 
2267     standards = resources_list_standards();
2268 
2269     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2270         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2271         rc++;
2272     }
2273 
2274     if (list_stonith_agents(NULL) > 0) {
2275         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2276         rc++;
2277     }
2278 
2279     g_list_free_full(standards, free);
2280     return rc;
2281 }
2282 
2283 /*!
2284  * \internal
2285  * \brief Create an executor API object
2286  *
2287  * \param[out] api       Will be set to newly created API object (it is the
2288  *                       caller's responsibility to free this value with
2289  *                       lrmd_api_delete() if this function succeeds)
2290  * \param[in]  nodename  If the object will be used for a remote connection,
2291  *                       the node name to use in cluster for remote executor
2292  * \param[in]  server    If the object will be used for a remote connection,
2293  *                       the resolvable host name to connect to
2294  * \param[in]  port      If the object will be used for a remote connection,
2295  *                       port number on \p server to connect to
2296  *
2297  * \return Standard Pacemaker return code
2298  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2299  *       value will be used for both. If the caller leaves both NULL, an API
2300  *       object will be created for a local executor connection.
2301  */
2302 int
2303 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2304 {
2305     lrmd_private_t *pvt = NULL;
2306 
2307     if (api == NULL) {
2308         return EINVAL;
2309     }
2310     *api = NULL;
2311 
2312     // Allocate all memory needed
2313 
2314     *api = calloc(1, sizeof(lrmd_t));
2315     if (*api == NULL) {
2316         return ENOMEM;
2317     }
2318 
2319     pvt = calloc(1, sizeof(lrmd_private_t));
2320     if (pvt == NULL) {
2321         lrmd_api_delete(*api);
2322         *api = NULL;
2323         return ENOMEM;
2324     }
2325     (*api)->lrmd_private = pvt;
2326 
2327     // @TODO Do we need to do this for local connections?
2328     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2329 
2330     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2331 
2332     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2333         lrmd_api_delete(*api);
2334         *api = NULL;
2335         return ENOMEM;
2336     }
2337 
2338     // Set methods
2339     (*api)->cmds->connect = lrmd_api_connect;
2340     (*api)->cmds->connect_async = lrmd_api_connect_async;
2341     (*api)->cmds->is_connected = lrmd_api_is_connected;
2342     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2343     (*api)->cmds->disconnect = lrmd_api_disconnect;
2344     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2345     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2346     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2347     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2348     (*api)->cmds->set_callback = lrmd_api_set_callback;
2349     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2350     (*api)->cmds->exec = lrmd_api_exec;
2351     (*api)->cmds->cancel = lrmd_api_cancel;
2352     (*api)->cmds->list_agents = lrmd_api_list_agents;
2353     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2354     (*api)->cmds->list_standards = lrmd_api_list_standards;
2355     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2356     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2357 
2358     if ((nodename == NULL) && (server == NULL)) {
2359         pvt->type = pcmk__client_ipc;
2360     } else {
2361 #ifdef HAVE_GNUTLS_GNUTLS_H
2362         if (nodename == NULL) {
2363             nodename = server;
2364         } else if (server == NULL) {
2365             server = nodename;
2366         }
2367         pvt->type = pcmk__client_tls;
2368         pvt->remote_nodename = strdup(nodename);
2369         pvt->server = strdup(server);
2370         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2371             lrmd_api_delete(*api);
2372             *api = NULL;
2373             return ENOMEM;
2374         }
2375         pvt->port = port;
2376         if (pvt->port == 0) {
2377             pvt->port = crm_default_remote_port();
2378         }
2379 #else
2380         crm_err("Cannot communicate with Pacemaker Remote "
2381                 "because GnuTLS is not enabled for this build");
2382         lrmd_api_delete(*api);
2383         *api = NULL;
2384         return EOPNOTSUPP;
2385 #endif
2386     }
2387     return pcmk_rc_ok;
2388 }
2389 
2390 lrmd_t *
2391 lrmd_api_new(void)
     /*  */
2392 {
2393     lrmd_t *api = NULL;
2394 
2395     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2396     return api;
2397 }
2398 
2399 lrmd_t *
2400 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2401 {
2402     lrmd_t *api = NULL;
2403 
2404     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2405     return api;
2406 }
2407 
2408 void
2409 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2410 {
2411     if (lrmd == NULL) {
2412         return;
2413     }
2414     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2415         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2416             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2417         }
2418         free(lrmd->cmds);
2419     }
2420     if (lrmd->lrmd_private != NULL) {
2421         lrmd_private_t *native = lrmd->lrmd_private;
2422 
2423 #ifdef HAVE_GNUTLS_GNUTLS_H
2424         free(native->server);
2425 #endif
2426         free(native->remote_nodename);
2427         free(native->remote);
2428         free(native->token);
2429         free(native->peer_version);
2430         free(lrmd->lrmd_private);
2431     }
2432     free(lrmd);
2433 }
2434 
2435 struct metadata_cb {
2436      void (*callback)(int pid, const pcmk__action_result_t *result,
2437                       void *user_data);
2438      void *user_data;
2439 };
2440 
2441 /*!
2442  * \internal
2443  * \brief Process asynchronous metadata completion
2444  *
2445  * \param[in,out] action  Metadata action that completed
2446  */
2447 static void
2448 metadata_complete(svc_action_t *action)
     /*  */
2449 {
2450     struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2451     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2452 
2453     pcmk__set_result(&result, action->rc, action->status,
2454                      services__exit_reason(action));
2455     pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2456 
2457     metadata_cb->callback(0, &result, metadata_cb->user_data);
2458     result.action_stdout = NULL; // Prevent free, because action owns it
2459     result.action_stderr = NULL; // Prevent free, because action owns it
2460     pcmk__reset_result(&result);
2461     free(metadata_cb);
2462 }
2463 
2464 /*!
2465  * \internal
2466  * \brief Retrieve agent metadata asynchronously
2467  *
2468  * \param[in]     rsc        Resource agent specification
2469  * \param[in]     callback   Function to call with result (this will always be
2470  *                           called, whether by this function directly or later
2471  *                           via the main loop, and on success the metadata will
2472  *                           be in its result argument's action_stdout)
2473  * \param[in,out] user_data  User data to pass to callback
2474  *
2475  * \return Standard Pacemaker return code
2476  * \note This function is not a lrmd_api_operations_t method because it does not
2477  *       need an lrmd_t object and does not go through the executor, but
2478  *       executes the agent directly.
2479  */
2480 int
2481 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
     /*  */
2482                      void (*callback)(int pid,
2483                                       const pcmk__action_result_t *result,
2484                                       void *user_data),
2485                      void *user_data)
2486 {
2487     svc_action_t *action = NULL;
2488     struct metadata_cb *metadata_cb = NULL;
2489     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2490 
2491     CRM_CHECK(callback != NULL, return EINVAL);
2492 
2493     if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2494         pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2495                          PCMK_EXEC_ERROR_FATAL,
2496                          "Invalid resource specification");
2497         callback(0, &result, user_data);
2498         pcmk__reset_result(&result);
2499         return EINVAL;
2500     }
2501 
2502     if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2503         return stonith__metadata_async(rsc->type,
2504                                        PCMK_DEFAULT_METADATA_TIMEOUT_MS / 1000,
2505                                        callback, user_data);
2506     }
2507 
2508     action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2509                                               rsc->standard, rsc->provider,
2510                                               rsc->type,
2511                                               PCMK_ACTION_META_DATA, 0,
2512                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
2513                                               NULL, 0);
2514     if (action == NULL) {
2515         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2516                          "Out of memory");
2517         callback(0, &result, user_data);
2518         pcmk__reset_result(&result);
2519         return ENOMEM;
2520     }
2521     if (action->rc != PCMK_OCF_UNKNOWN) {
2522         pcmk__set_result(&result, action->rc, action->status,
2523                          services__exit_reason(action));
2524         callback(0, &result, user_data);
2525         pcmk__reset_result(&result);
2526         services_action_free(action);
2527         return EINVAL;
2528     }
2529 
2530     action->cb_data = calloc(1, sizeof(struct metadata_cb));
2531     if (action->cb_data == NULL) {
2532         services_action_free(action);
2533         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2534                          "Out of memory");
2535         callback(0, &result, user_data);
2536         pcmk__reset_result(&result);
2537         return ENOMEM;
2538     }
2539 
2540     metadata_cb = (struct metadata_cb *) action->cb_data;
2541     metadata_cb->callback = callback;
2542     metadata_cb->user_data = user_data;
2543     if (!services_action_async(action, metadata_complete)) {
2544         services_action_free(action);
2545         return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2546     }
2547 
2548     // The services library has taken responsibility for action
2549     return pcmk_rc_ok;
2550 }
2551 
2552 /*!
2553  * \internal
2554  * \brief Set the result of an executor event
2555  *
2556  * \param[in,out] event        Executor event to set
2557  * \param[in]     rc           OCF exit status of event
2558  * \param[in]     op_status    Executor status of event
2559  * \param[in]     exit_reason  Human-friendly description of event
2560  */
2561 void
2562 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2563                  const char *exit_reason)
2564 {
2565     if (event == NULL) {
2566         return;
2567     }
2568 
2569     event->rc = rc;
2570     event->op_status = op_status;
2571 
2572     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
2573     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2574 }
2575 
2576 /*!
2577  * \internal
2578  * \brief Clear an executor event's exit reason, output, and error output
2579  *
2580  * \param[in,out] event  Executor event to reset
2581  */
2582 void
2583 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2584 {
2585     if (event == NULL) {
2586         return;
2587     }
2588 
2589     free((void *) event->exit_reason);
2590     event->exit_reason = NULL;
2591 
2592     free((void *) event->output);
2593     event->output = NULL;
2594 }
2595 
2596 /*!
2597  * \internal
2598  * \brief Get the uptime of a remote resource connection
2599  *
2600  * When the cluster connects to a remote resource, part of that resource's
2601  * handshake includes the uptime of the remote resource's connection.  This
2602  * uptime is stored in the lrmd_t object.
2603  *
2604  * \return The connection's uptime, or -1 if unknown
2605  */
2606 time_t
2607 lrmd__uptime(lrmd_t *lrmd)
     /*  */
2608 {
2609     lrmd_private_t *native = lrmd->lrmd_private;
2610 
2611     if (native->remote == NULL) {
2612         return -1;
2613     } else {
2614         return native->remote->uptime;
2615     }
2616 }
2617 
2618 const char *
2619 lrmd__node_start_state(lrmd_t *lrmd)
     /*  */
2620 {
2621     lrmd_private_t *native = lrmd->lrmd_private;
2622 
2623     if (native->remote == NULL) {
2624         return NULL;
2625     } else {
2626         return native->remote->start_state;
2627     }
2628 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
