lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
remote_executor_connected
handle_remote_msg
process_pending_notifies
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd__remote_send_xml
read_remote_reply
send_remote_message
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
lrmd__validate_remote_settings
lrmd_handshake_hello_msg
process_lrmd_handshake_reply
lrmd_handshake
lrmd_handshake_async
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
read_gnutls_key
key_is_cached
key_cache_expired
clear_key_cache
get_cached_key
cache_key
get_remote_key
lrmd__init_remote_key
report_async_connection_result
tls_handshake_failed
tls_handshake_succeeded
tls_client_handshake
add_tls_to_mainloop
try_handshake_cb
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd__new
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
metadata_complete
lrmd__metadata_async
lrmd__set_result
lrmd__reset_result
lrmd__uptime
lrmd__node_start_state
   1 /*
   2  * Copyright 2012-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/common/tls_internal.h>
  36 #include <crm/common/xml.h>
  37 
  38 #include <crm/stonith-ng.h>
  39 #include <crm/fencing/internal.h>   // stonith__*
  40 
  41 #include <gnutls/gnutls.h>
  42 #include <sys/socket.h>
  43 #include <netinet/in.h>
  44 #include <netinet/ip.h>
  45 #include <arpa/inet.h>
  46 #include <netdb.h>
  47 
  48 #define MAX_TLS_RECV_WAIT 10000
  49 
  50 CRM_TRACE_INIT_DATA(lrmd);
  51 
  52 static int lrmd_api_disconnect(lrmd_t * lrmd);
  53 static int lrmd_api_is_connected(lrmd_t * lrmd);
  54 
  55 /* IPC proxy functions */
  56 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  57 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  58 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  59 
  60 // GnuTLS client handshake timeout in seconds
  61 #define TLS_HANDSHAKE_TIMEOUT 5
  62 
  63 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  64 static int global_remote_msg_id = 0;
  65 static void lrmd_tls_connection_destroy(gpointer userdata);
  66 static int add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake);
  67 
  68 typedef struct lrmd_private_s {
  69     uint64_t type;
  70     char *token;
  71     mainloop_io_t *source;
  72 
  73     /* IPC parameters */
  74     crm_ipc_t *ipc;
  75 
  76     pcmk__remote_t *remote;
  77 
  78     /* Extra TLS parameters */
  79     char *remote_nodename;
  80     char *server;
  81     int port;
  82     pcmk__tls_t *tls;
  83 
  84     /* while the async connection is occurring, this is the id
  85      * of the connection timeout timer. */
  86     int async_timer;
  87     int sock;
  88     /* since tls requires a round trip across the network for a
  89      * request/reply, there are times where we just want to be able
  90      * to send a request from the client and not wait around (or even care
  91      * about) what the reply is. */
  92     int expected_late_replies;
  93     GList *pending_notify;
  94     crm_trigger_t *process_notify;
  95     crm_trigger_t *handshake_trigger;
  96 
  97     lrmd_event_callback callback;
  98 
  99     /* Internal IPC proxy msg passing for remote guests */
 100     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 101     void *proxy_callback_userdata;
 102     char *peer_version;
 103 } lrmd_private_t;
 104 
 105 static int process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native);
 106 static void report_async_connection_result(lrmd_t * lrmd, int rc);
 107 
 108 static lrmd_list_t *
 109 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 110 {
 111     lrmd_list_t *p, *end;
 112 
 113     p = pcmk__assert_alloc(1, sizeof(lrmd_list_t));
 114     p->val = strdup(value);
 115 
 116     end = head;
 117     while (end && end->next) {
 118         end = end->next;
 119     }
 120 
 121     if (end) {
 122         end->next = p;
 123     } else {
 124         head = p;
 125     }
 126 
 127     return head;
 128 }
 129 
 130 void
 131 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 132 {
 133     lrmd_list_t *p;
 134 
 135     while (head) {
 136         char *val = (char *)head->val;
 137 
 138         p = head->next;
 139         free(val);
 140         free(head);
 141         head = p;
 142     }
 143 }
 144 
 145 lrmd_key_value_t *
 146 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 147 {
 148     lrmd_key_value_t *p, *end;
 149 
 150     p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t));
 151     p->key = strdup(key);
 152     p->value = strdup(value);
 153 
 154     end = head;
 155     while (end && end->next) {
 156         end = end->next;
 157     }
 158 
 159     if (end) {
 160         end->next = p;
 161     } else {
 162         head = p;
 163     }
 164 
 165     return head;
 166 }
 167 
 168 void
 169 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 170 {
 171     lrmd_key_value_t *p;
 172 
 173     while (head) {
 174         p = head->next;
 175         free(head->key);
 176         free(head->value);
 177         free(head);
 178         head = p;
 179     }
 180 }
 181 
 182 /*!
 183  * \brief Create a new lrmd_event_data_t object
 184  *
 185  * \param[in] rsc_id       ID of resource involved in event
 186  * \param[in] task         Action name
 187  * \param[in] interval_ms  Action interval
 188  *
 189  * \return Newly allocated and initialized lrmd_event_data_t
 190  * \note This functions asserts on memory errors, so the return value is
 191  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 192  *       result with lrmd_free_event().
 193  */
 194 lrmd_event_data_t *
 195 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 196 {
 197     lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 198 
 199     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 200     event->rsc_id = pcmk__str_copy(rsc_id);
 201     event->op_type = pcmk__str_copy(task);
 202     event->interval_ms = interval_ms;
 203     return event;
 204 }
 205 
 206 lrmd_event_data_t *
 207 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 208 {
 209     lrmd_event_data_t *copy = NULL;
 210 
 211     copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 212 
 213     copy->type = event->type;
 214 
 215     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 216     copy->rsc_id = pcmk__str_copy(event->rsc_id);
 217     copy->op_type = pcmk__str_copy(event->op_type);
 218     copy->user_data = pcmk__str_copy(event->user_data);
 219     copy->output = pcmk__str_copy(event->output);
 220     copy->remote_nodename = pcmk__str_copy(event->remote_nodename);
 221     copy->exit_reason = pcmk__str_copy(event->exit_reason);
 222 
 223     copy->call_id = event->call_id;
 224     copy->timeout = event->timeout;
 225     copy->interval_ms = event->interval_ms;
 226     copy->start_delay = event->start_delay;
 227     copy->rsc_deleted = event->rsc_deleted;
 228     copy->rc = event->rc;
 229     copy->op_status = event->op_status;
 230     copy->t_run = event->t_run;
 231     copy->t_rcchange = event->t_rcchange;
 232     copy->exec_time = event->exec_time;
 233     copy->queue_time = event->queue_time;
 234     copy->connection_rc = event->connection_rc;
 235     copy->params = pcmk__str_table_dup(event->params);
 236 
 237     return copy;
 238 }
 239 
 240 /*!
 241  * \brief Free an executor event
 242  *
 243  * \param[in,out]  Executor event object to free
 244  */
 245 void
 246 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 247 {
 248     if (event == NULL) {
 249         return;
 250     }
 251     // @TODO Why are these const char *?
 252     free((void *) event->rsc_id);
 253     free((void *) event->op_type);
 254     free((void *) event->user_data);
 255     free((void *) event->remote_nodename);
 256     lrmd__reset_result(event);
 257     if (event->params != NULL) {
 258         g_hash_table_destroy(event->params);
 259     }
 260     free(event);
 261 }
 262 
 263 static void
 264 lrmd_dispatch_internal(gpointer data, gpointer user_data)
     /*  */
 265 {
 266     xmlNode *msg = data;
 267     lrmd_t *lrmd = user_data;
 268 
 269     const char *type;
 270     const char *proxy_session = crm_element_value(msg,
 271                                                   PCMK__XA_LRMD_IPC_SESSION);
 272     lrmd_private_t *native = lrmd->lrmd_private;
 273     lrmd_event_data_t event = { 0, };
 274 
 275     if (proxy_session != NULL) {
 276         /* this is proxy business */
 277         lrmd_internal_proxy_dispatch(lrmd, msg);
 278         return;
 279     } else if (!native->callback) {
 280         /* no callback set */
 281         crm_trace("notify event received but client has not set callback");
 282         return;
 283     }
 284 
 285     event.remote_nodename = native->remote_nodename;
 286     type = crm_element_value(msg, PCMK__XA_LRMD_OP);
 287     crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
 288     event.rsc_id = crm_element_value(msg, PCMK__XA_LRMD_RSC_ID);
 289 
 290     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 291         event.type = lrmd_event_register;
 292     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 293         event.type = lrmd_event_unregister;
 294     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 295         int rc = 0;
 296         int exec_time = 0;
 297         int queue_time = 0;
 298         time_t epoch = 0;
 299 
 300         crm_element_value_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
 301         crm_element_value_ms(msg, PCMK__XA_LRMD_RSC_INTERVAL,
 302                              &event.interval_ms);
 303         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
 304                               &event.start_delay);
 305 
 306         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc);
 307         event.rc = (enum ocf_exitcode) rc;
 308 
 309         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS,
 310                               &event.op_status);
 311         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_DELETED,
 312                               &event.rsc_deleted);
 313 
 314         crm_element_value_epoch(msg, PCMK__XA_LRMD_RUN_TIME, &epoch);
 315         event.t_run = epoch;
 316 
 317         crm_element_value_epoch(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &epoch);
 318         event.t_rcchange = epoch;
 319 
 320         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time);
 321         CRM_LOG_ASSERT(exec_time >= 0);
 322         event.exec_time = QB_MAX(0, exec_time);
 323 
 324         crm_element_value_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time);
 325         CRM_LOG_ASSERT(queue_time >= 0);
 326         event.queue_time = QB_MAX(0, queue_time);
 327 
 328         event.op_type = crm_element_value(msg, PCMK__XA_LRMD_RSC_ACTION);
 329         event.user_data = crm_element_value(msg,
 330                                             PCMK__XA_LRMD_RSC_USERDATA_STR);
 331         event.type = lrmd_event_exec_complete;
 332 
 333         /* output and exit_reason may be freed by a callback */
 334         event.output = crm_element_value_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
 335         lrmd__set_result(&event, event.rc, event.op_status,
 336                          crm_element_value(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));
 337 
 338         event.params = xml2list(msg);
 339     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 340         event.type = lrmd_event_new_client;
 341     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 342         event.type = lrmd_event_poke;
 343     } else {
 344         return;
 345     }
 346 
 347     crm_trace("op %s notify event received", type);
 348     native->callback(&event);
 349 
 350     if (event.params) {
 351         g_hash_table_destroy(event.params);
 352     }
 353     lrmd__reset_result(&event);
 354 }
 355 
 356 // \return Always 0, to indicate that IPC mainloop source should be kept
 357 static int
 358 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 359 {
 360     lrmd_t *lrmd = userdata;
 361     lrmd_private_t *native = lrmd->lrmd_private;
 362 
 363     if (native->callback != NULL) {
 364         xmlNode *msg = pcmk__xml_parse(buffer);
 365 
 366         lrmd_dispatch_internal(msg, lrmd);
 367         pcmk__xml_free(msg);
 368     }
 369     return 0;
 370 }
 371 
 372 static void
 373 lrmd_free_xml(gpointer userdata)
     /*  */
 374 {
 375     pcmk__xml_free((xmlNode *) userdata);
 376 }
 377 
 378 static bool
 379 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 380 {
 381     lrmd_private_t *native = lrmd->lrmd_private;
 382 
 383     return (native->remote->tls_session != NULL);
 384 }
 385 
 386 static void
 387 handle_remote_msg(xmlNode *xml, lrmd_t *lrmd)
     /*  */
 388 {
 389     lrmd_private_t *native = lrmd->lrmd_private;
 390     const char *msg_type = NULL;
 391 
 392     msg_type = crm_element_value(xml, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 393     if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 394         lrmd_dispatch_internal(xml, lrmd);
 395     } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 396         const char *op = crm_element_value(xml, PCMK__XA_LRMD_OP);
 397 
 398         if (native->expected_late_replies > 0) {
 399             native->expected_late_replies--;
 400 
 401             /* The register op message we get as a response to lrmd_handshake_async
 402              * is a reply, so we have to handle that here.
 403              */
 404             if (pcmk__str_eq(op, "register", pcmk__str_casei)) {
 405                 int rc = process_lrmd_handshake_reply(xml, native);
 406                 report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
 407             }
 408         } else {
 409             int reply_id = 0;
 410             crm_element_value_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
 411             /* if this happens, we want to know about it */
 412             crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 413         }
 414     }
 415 }
 416 
 417 /*!
 418  * \internal
 419  * \brief Notify trigger handler
 420  *
 421  * \param[in,out] userdata API connection
 422  *
 423  * \return Always return G_SOURCE_CONTINUE to leave this trigger handler in the
 424  *         mainloop
 425  */
 426 static int
 427 process_pending_notifies(gpointer userdata)
     /*  */
 428 {
 429     lrmd_t *lrmd = userdata;
 430     lrmd_private_t *native = lrmd->lrmd_private;
 431 
 432     if (native->pending_notify == NULL) {
 433         return G_SOURCE_CONTINUE;
 434     }
 435 
 436     crm_trace("Processing pending notifies");
 437     g_list_foreach(native->pending_notify, lrmd_dispatch_internal, lrmd);
 438     g_list_free_full(native->pending_notify, lrmd_free_xml);
 439     native->pending_notify = NULL;
 440     return G_SOURCE_CONTINUE;
 441 }
 442 
 443 /*!
 444  * \internal
 445  * \brief TLS dispatch function for file descriptor sources
 446  *
 447  * \param[in,out] userdata  API connection
 448  *
 449  * \return -1 on error to remove the source from the mainloop, or 0 otherwise
 450  *         to leave it in the mainloop
 451  */
 452 static int
 453 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 454 {
 455     lrmd_t *lrmd = userdata;
 456     lrmd_private_t *native = lrmd->lrmd_private;
 457     xmlNode *xml = NULL;
 458     int rc = pcmk_rc_ok;
 459 
 460     if (!remote_executor_connected(lrmd)) {
 461         crm_trace("TLS dispatch triggered after disconnect");
 462         return -1;
 463     }
 464 
 465     crm_trace("TLS dispatch triggered");
 466 
 467     rc = pcmk__remote_ready(native->remote, 0);
 468     if (rc == pcmk_rc_ok) {
 469         rc = pcmk__read_remote_message(native->remote, -1);
 470     }
 471 
 472     if (rc != pcmk_rc_ok && rc != ETIME) {
 473         crm_info("Lost %s executor connection while reading data",
 474                  (native->remote_nodename? native->remote_nodename : "local"));
 475         lrmd_tls_disconnect(lrmd);
 476         return -1;
 477     }
 478 
 479     /* If rc is ETIME, there was nothing to read but we may already have a
 480      * full message in the buffer
 481      */
 482     xml = pcmk__remote_message_xml(native->remote);
 483 
 484     if (xml == NULL) {
 485         return 0;
 486     }
 487 
 488     handle_remote_msg(xml, lrmd);
 489     pcmk__xml_free(xml);
 490     return 0;
 491 }
 492 
 493 /* Not used with mainloop */
 494 int
 495 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 496 {
 497     lrmd_private_t *native = lrmd->lrmd_private;
 498 
 499     switch (native->type) {
 500         case pcmk__client_ipc:
 501             return crm_ipc_ready(native->ipc);
 502 
 503         case pcmk__client_tls:
 504             if (native->pending_notify) {
 505                 return 1;
 506             } else {
 507                 int rc = pcmk__remote_ready(native->remote, 0);
 508 
 509                 switch (rc) {
 510                     case pcmk_rc_ok:
 511                         return 1;
 512                     case ETIME:
 513                         return 0;
 514                     default:
 515                         return pcmk_rc2legacy(rc);
 516                 }
 517             }
 518         default:
 519             crm_err("Unsupported executor connection type (bug?): %d",
 520                     native->type);
 521             return -EPROTONOSUPPORT;
 522     }
 523 }
 524 
 525 /* Not used with mainloop */
 526 bool
 527 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 528 {
 529     lrmd_private_t *private = NULL;
 530 
 531     pcmk__assert(lrmd != NULL);
 532 
 533     private = lrmd->lrmd_private;
 534     switch (private->type) {
 535         case pcmk__client_ipc:
 536             while (crm_ipc_ready(private->ipc)) {
 537                 if (crm_ipc_read(private->ipc) > 0) {
 538                     const char *msg = crm_ipc_buffer(private->ipc);
 539 
 540                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 541                 }
 542             }
 543             break;
 544         case pcmk__client_tls:
 545             lrmd_tls_dispatch(lrmd);
 546             break;
 547         default:
 548             crm_err("Unsupported executor connection type (bug?): %d",
 549                     private->type);
 550     }
 551 
 552     if (lrmd_api_is_connected(lrmd) == FALSE) {
 553         crm_err("Connection closed");
 554         return FALSE;
 555     }
 556 
 557     return TRUE;
 558 }
 559 
 560 static xmlNode *
 561 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 562                enum lrmd_call_options options)
 563 {
 564     xmlNode *op_msg = NULL;
 565 
 566     CRM_CHECK(token != NULL, return NULL);
 567 
 568     op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
 569     crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
 570     crm_xml_add(op_msg, PCMK__XA_LRMD_OP, op);
 571     crm_xml_add_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
 572     crm_xml_add_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);
 573 
 574     if (data != NULL) {
 575         xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA);
 576 
 577         pcmk__xml_copy(wrapper, data);
 578     }
 579 
 580     crm_trace("Created executor %s command with call options %.8lx (%d)",
 581               op, (long)options, options);
 582     return op_msg;
 583 }
 584 
 585 static void
 586 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 587 {
 588     lrmd_t *lrmd = userdata;
 589     lrmd_private_t *native = lrmd->lrmd_private;
 590 
 591     switch (native->type) {
 592         case pcmk__client_ipc:
 593             crm_info("Disconnected from local executor");
 594             break;
 595         case pcmk__client_tls:
 596             crm_info("Disconnected from remote executor on %s",
 597                      native->remote_nodename);
 598             break;
 599         default:
 600             crm_err("Unsupported executor connection type %d (bug?)",
 601                     native->type);
 602     }
 603 
 604     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 605     native->ipc = NULL;
 606     native->source = NULL;
 607 
 608     if (native->callback) {
 609         lrmd_event_data_t event = { 0, };
 610         event.type = lrmd_event_disconnect;
 611         event.remote_nodename = native->remote_nodename;
 612         native->callback(&event);
 613     }
 614 }
 615 
 616 static void
 617 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 618 {
 619     lrmd_t *lrmd = userdata;
 620     lrmd_private_t *native = lrmd->lrmd_private;
 621 
 622     crm_info("TLS connection destroyed");
 623 
 624     if (native->remote->tls_session) {
 625         gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
 626         gnutls_deinit(native->remote->tls_session);
 627         native->remote->tls_session = NULL;
 628     }
 629     if (native->tls) {
 630         pcmk__free_tls(native->tls);
 631         native->tls = NULL;
 632     }
 633     if (native->sock) {
 634         close(native->sock);
 635     }
 636     if (native->process_notify) {
 637         mainloop_destroy_trigger(native->process_notify);
 638         native->process_notify = NULL;
 639     }
 640     if (native->pending_notify) {
 641         g_list_free_full(native->pending_notify, lrmd_free_xml);
 642         native->pending_notify = NULL;
 643     }
 644     if (native->handshake_trigger != NULL) {
 645         mainloop_destroy_trigger(native->handshake_trigger);
 646         native->handshake_trigger = NULL;
 647     }
 648 
 649     free(native->remote->buffer);
 650     free(native->remote->start_state);
 651     native->remote->buffer = NULL;
 652     native->remote->start_state = NULL;
 653     native->source = 0;
 654     native->sock = 0;
 655 
 656     if (native->callback) {
 657         lrmd_event_data_t event = { 0, };
 658         event.remote_nodename = native->remote_nodename;
 659         event.type = lrmd_event_disconnect;
 660         native->callback(&event);
 661     }
 662     return;
 663 }
 664 
 665 // \return Standard Pacemaker return code
 666 int
 667 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 668                       const char *msg_type)
 669 {
 670     crm_xml_add_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
 671     crm_xml_add(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
 672     return pcmk__remote_send_xml(session, msg);
 673 }
 674 
 675 // \return Standard Pacemaker return code
 676 static int
 677 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 678                   xmlNode **reply)
 679 {
 680     lrmd_private_t *native = lrmd->lrmd_private;
 681     time_t start = time(NULL);
 682     const char *msg_type = NULL;
 683     int reply_id = 0;
 684     int remaining_timeout = 0;
 685     int rc = pcmk_rc_ok;
 686 
 687     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 688      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 689     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 690         total_timeout = MAX_TLS_RECV_WAIT;
 691     }
 692 
 693     for (*reply = NULL; *reply == NULL; ) {
 694 
 695         *reply = pcmk__remote_message_xml(native->remote);
 696         if (*reply == NULL) {
 697             /* read some more off the tls buffer if we still have time left. */
 698             if (remaining_timeout) {
 699                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 700             } else {
 701                 remaining_timeout = total_timeout;
 702             }
 703             if (remaining_timeout <= 0) {
 704                 return ETIME;
 705             }
 706 
 707             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 708             if (rc != pcmk_rc_ok) {
 709                 return rc;
 710             }
 711 
 712             *reply = pcmk__remote_message_xml(native->remote);
 713             if (*reply == NULL) {
 714                 return ENOMSG;
 715             }
 716         }
 717 
 718         crm_element_value_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
 719         msg_type = crm_element_value(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 720 
 721         if (!msg_type) {
 722             crm_err("Empty msg type received while waiting for reply");
 723             pcmk__xml_free(*reply);
 724             *reply = NULL;
 725         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 726             /* got a notify while waiting for reply, trigger the notify to be processed later */
 727             crm_info("queueing notify");
 728             native->pending_notify = g_list_append(native->pending_notify, *reply);
 729             if (native->process_notify) {
 730                 crm_info("notify trigger set.");
 731                 mainloop_set_trigger(native->process_notify);
 732             }
 733             *reply = NULL;
 734         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 735             /* msg isn't a reply, make some noise */
 736             crm_err("Expected a reply, got %s", msg_type);
 737             pcmk__xml_free(*reply);
 738             *reply = NULL;
 739         } else if (reply_id != expected_reply_id) {
 740             if (native->expected_late_replies > 0) {
 741                 native->expected_late_replies--;
 742             } else {
 743                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 744             }
 745             pcmk__xml_free(*reply);
 746             *reply = NULL;
 747         }
 748     }
 749 
 750     if (native->remote->buffer && native->process_notify) {
 751         mainloop_set_trigger(native->process_notify);
 752     }
 753 
 754     return rc;
 755 }
 756 
 757 // \return Standard Pacemaker return code
 758 static int
 759 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 760 {
 761     int rc = pcmk_rc_ok;
 762     lrmd_private_t *native = lrmd->lrmd_private;
 763 
 764     global_remote_msg_id++;
 765     if (global_remote_msg_id <= 0) {
 766         global_remote_msg_id = 1;
 767     }
 768 
 769     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 770                                "request");
 771     if (rc != pcmk_rc_ok) {
 772         crm_err("Disconnecting because TLS message could not be sent to "
 773                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 774         lrmd_tls_disconnect(lrmd);
 775     }
 776     return rc;
 777 }
 778 
 779 static int
 780 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 781 {
 782     int rc = 0;
 783     xmlNode *xml = NULL;
 784 
 785     if (!remote_executor_connected(lrmd)) {
 786         return -ENOTCONN;
 787     }
 788 
 789     rc = send_remote_message(lrmd, msg);
 790     if (rc != pcmk_rc_ok) {
 791         return pcmk_rc2legacy(rc);
 792     }
 793 
 794     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 795     if (rc != pcmk_rc_ok) {
 796         crm_err("Disconnecting remote after request %d reply not received: %s "
 797                 QB_XS " rc=%d timeout=%dms",
 798                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 799         lrmd_tls_disconnect(lrmd);
 800     }
 801 
 802     if (reply) {
 803         *reply = xml;
 804     } else {
 805         pcmk__xml_free(xml);
 806     }
 807 
 808     return pcmk_rc2legacy(rc);
 809 }
 810 
 811 static int
 812 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 813 {
 814     int rc = pcmk_ok;
 815     lrmd_private_t *native = lrmd->lrmd_private;
 816 
 817     switch (native->type) {
 818         case pcmk__client_ipc:
 819             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 820             break;
 821         case pcmk__client_tls:
 822             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 823             break;
 824         default:
 825             crm_err("Unsupported executor connection type (bug?): %d",
 826                     native->type);
 827             rc = -EPROTONOSUPPORT;
 828     }
 829 
 830     return rc;
 831 }
 832 
 833 static int
 834 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 835 {
 836     int rc = pcmk_ok;
 837     lrmd_private_t *native = lrmd->lrmd_private;
 838 
 839     switch (native->type) {
 840         case pcmk__client_ipc:
 841             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 842             break;
 843         case pcmk__client_tls:
 844             rc = send_remote_message(lrmd, msg);
 845             if (rc == pcmk_rc_ok) {
 846                 /* we don't want to wait around for the reply, but
 847                  * since the request/reply protocol needs to behave the same
 848                  * as libqb, a reply will eventually come later anyway. */
 849                 native->expected_late_replies++;
 850             }
 851             rc = pcmk_rc2legacy(rc);
 852             break;
 853         default:
 854             crm_err("Unsupported executor connection type (bug?): %d",
 855                     native->type);
 856             rc = -EPROTONOSUPPORT;
 857     }
 858 
 859     return rc;
 860 }
 861 
 862 static int
 863 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 864 {
 865     lrmd_private_t *native = lrmd->lrmd_private;
 866 
 867     switch (native->type) {
 868         case pcmk__client_ipc:
 869             return crm_ipc_connected(native->ipc);
 870         case pcmk__client_tls:
 871             return remote_executor_connected(lrmd);
 872         default:
 873             crm_err("Unsupported executor connection type (bug?): %d",
 874                     native->type);
 875             return 0;
 876     }
 877 }
 878 
 879 /*!
 880  * \internal
 881  * \brief Send a prepared API command to the executor
 882  *
 883  * \param[in,out] lrmd          Existing connection to the executor
 884  * \param[in]     op            Name of API command to send
 885  * \param[in]     data          Command data XML to add to the sent command
 886  * \param[out]    output_data   If expecting a reply, it will be stored here
 887  * \param[in]     timeout       Timeout in milliseconds (if 0, defaults to
 888  *                              a sensible value per the type of connection,
 889  *                              standard vs. pacemaker remote);
 890  *                              also propagated to the command XML
 891  * \param[in]     call_options  Call options to pass to server when sending
 892  * \param[in]     expect_reply  If true, wait for a reply from the server;
 893  *                              must be true for IPC (as opposed to TLS) clients
 894  *
 895  * \return pcmk_ok on success, -errno on error
 896  */
 897 static int
 898 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 899                   xmlNode **output_data, int timeout,
 900                   enum lrmd_call_options options, bool expect_reply)
 901 {
 902     int rc = pcmk_ok;
 903     lrmd_private_t *native = lrmd->lrmd_private;
 904     xmlNode *op_msg = NULL;
 905     xmlNode *op_reply = NULL;
 906 
 907     if (!lrmd_api_is_connected(lrmd)) {
 908         return -ENOTCONN;
 909     }
 910 
 911     if (op == NULL) {
 912         crm_err("No operation specified");
 913         return -EINVAL;
 914     }
 915 
 916     CRM_LOG_ASSERT(native->token != NULL);
 917     crm_trace("Sending %s op to executor", op);
 918 
 919     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 920 
 921     if (op_msg == NULL) {
 922         return -EINVAL;
 923     }
 924 
 925     if (expect_reply) {
 926         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 927     } else {
 928         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 929         goto done;
 930     }
 931 
 932     if (rc < 0) {
 933         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 934         goto done;
 935 
 936     } else if (op_reply == NULL) {
 937         rc = -ENOMSG;
 938         goto done;
 939     }
 940 
 941     rc = pcmk_ok;
 942     crm_trace("%s op reply received", op);
 943     if (crm_element_value_int(op_reply, PCMK__XA_LRMD_RC, &rc) != 0) {
 944         rc = -ENOMSG;
 945         goto done;
 946     }
 947 
 948     crm_log_xml_trace(op_reply, "Reply");
 949 
 950     if (output_data) {
 951         *output_data = op_reply;
 952         op_reply = NULL;        /* Prevent subsequent free */
 953     }
 954 
 955   done:
 956     if (lrmd_api_is_connected(lrmd) == FALSE) {
 957         crm_err("Executor disconnected");
 958     }
 959 
 960     pcmk__xml_free(op_msg);
 961     pcmk__xml_free(op_reply);
 962     return rc;
 963 }
 964 
 965 static int
 966 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 967 {
 968     int rc;
 969     lrmd_private_t *native = lrmd->lrmd_private;
 970     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
 971 
 972     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
 973     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 974                            (native->type == pcmk__client_ipc));
 975     pcmk__xml_free(data);
 976 
 977     return rc < 0 ? rc : pcmk_ok;
 978 }
 979 
 980 // \return Standard Pacemaker return code
 981 int
 982 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 983 {
 984     int rc = pcmk_rc_ok;
 985     const char *value;
 986     lrmd_private_t *native = lrmd->lrmd_private;
 987     xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP);
 988 
 989     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
 990 
 991     value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
 992     if ((value) &&
 993         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 994        crm_xml_add(data, PCMK__XA_LRMD_WATCHDOG, value);
 995     }
 996 
 997     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 998                            (native->type == pcmk__client_ipc));
 999     pcmk__xml_free(data);
1000     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
1001 }
1002 
1003 static xmlNode *
1004 lrmd_handshake_hello_msg(const char *name, bool is_proxy)
     /*  */
1005 {
1006     xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
1007 
1008     crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
1009     crm_xml_add(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
1010     crm_xml_add(hello, PCMK__XA_LRMD_CLIENTNAME, name);
1011     crm_xml_add(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
1012 
1013     /* advertise that we are a proxy provider */
1014     if (is_proxy) {
1015         pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
1016     }
1017 
1018     return hello;
1019 }
1020 
1021 static int
1022 process_lrmd_handshake_reply(xmlNode *reply, lrmd_private_t *native)
     /*  */
1023 {
1024     int rc = pcmk_rc_ok;
1025     const char *version = crm_element_value(reply, PCMK__XA_LRMD_PROTOCOL_VERSION);
1026     const char *msg_type = crm_element_value(reply, PCMK__XA_LRMD_OP);
1027     const char *tmp_ticket = crm_element_value(reply, PCMK__XA_LRMD_CLIENTID);
1028     const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
1029     long long uptime = -1;
1030 
1031     crm_element_value_int(reply, PCMK__XA_LRMD_RC, &rc);
1032     rc = pcmk_legacy2rc(rc);
1033 
1034     /* The remote executor may add its uptime to the XML reply, which is useful
1035      * in handling transient attributes when the connection to the remote node
1036      * unexpectedly drops.  If no parameter is given, just default to -1.
1037      */
1038     crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
1039     native->remote->uptime = uptime;
1040 
1041     if (start_state) {
1042         native->remote->start_state = strdup(start_state);
1043     }
1044 
1045     if (rc == EPROTO) {
1046         crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
1047                 LRMD_PROTOCOL_VERSION, version);
1048         crm_log_xml_err(reply, "Protocol Error");
1049     } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
1050         crm_err("Invalid registration message: %s", msg_type);
1051         crm_log_xml_err(reply, "Bad reply");
1052         rc = EPROTO;
1053     } else if (tmp_ticket == NULL) {
1054         crm_err("No registration token provided");
1055         crm_log_xml_err(reply, "Bad reply");
1056         rc = EPROTO;
1057     } else {
1058         crm_trace("Obtained registration token: %s", tmp_ticket);
1059         native->token = strdup(tmp_ticket);
1060         native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1061         rc = pcmk_rc_ok;
1062     }
1063 
1064     return rc;
1065 }
1066 
1067 static int
1068 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
1069 {
1070     int rc = pcmk_rc_ok;
1071     lrmd_private_t *native = lrmd->lrmd_private;
1072     xmlNode *reply = NULL;
1073     xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
1074 
1075     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
1076 
1077     if (rc < 0) {
1078         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
1079         rc = ECOMM;
1080     } else if (reply == NULL) {
1081         crm_err("Did not receive registration reply");
1082         rc = EPROTO;
1083     } else {
1084         rc = process_lrmd_handshake_reply(reply, native);
1085     }
1086 
1087     pcmk__xml_free(reply);
1088     pcmk__xml_free(hello);
1089 
1090     if (rc != pcmk_rc_ok) {
1091         lrmd_api_disconnect(lrmd);
1092     }
1093 
1094     return rc;
1095 }
1096 
1097 static int
1098 lrmd_handshake_async(lrmd_t * lrmd, const char *name)
     /*  */
1099 {
1100     int rc = pcmk_rc_ok;
1101     lrmd_private_t *native = lrmd->lrmd_private;
1102     xmlNode *hello = lrmd_handshake_hello_msg(name, native->proxy_callback != NULL);
1103 
1104     rc = send_remote_message(lrmd, hello);
1105 
1106     if (rc == pcmk_rc_ok) {
1107         native->expected_late_replies++;
1108     } else {
1109         lrmd_api_disconnect(lrmd);
1110     }
1111 
1112     pcmk__xml_free(hello);
1113     return rc;
1114 }
1115 
1116 static int
1117 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1118 {
1119     int rc = pcmk_ok;
1120     lrmd_private_t *native = lrmd->lrmd_private;
1121 
1122     struct ipc_client_callbacks lrmd_callbacks = {
1123         .dispatch = lrmd_ipc_dispatch,
1124         .destroy = lrmd_ipc_connection_destroy
1125     };
1126 
1127     crm_info("Connecting to executor");
1128 
1129     if (fd) {
1130         /* No mainloop */
1131         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1132         if (native->ipc != NULL) {
1133             rc = pcmk__connect_generic_ipc(native->ipc);
1134             if (rc == pcmk_rc_ok) {
1135                 rc = pcmk__ipc_fd(native->ipc, fd);
1136             }
1137             if (rc != pcmk_rc_ok) {
1138                 crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
1139                 rc = -ENOTCONN;
1140             }
1141         }
1142     } else {
1143         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1144         native->ipc = mainloop_get_ipc_client(native->source);
1145     }
1146 
1147     if (native->ipc == NULL) {
1148         crm_debug("Could not connect to the executor API");
1149         rc = -ENOTCONN;
1150     }
1151 
1152     return rc;
1153 }
1154 
1155 static void
1156 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1157 {
1158     pcmk__assert((dest != NULL) && (source != NULL) && (source->data != NULL));
1159 
1160     dest->data = gnutls_malloc(source->size);
1161     pcmk__mem_assert(dest->data);
1162 
1163     memcpy(dest->data, source->data, source->size);
1164     dest->size = source->size;
1165 }
1166 
1167 static void
1168 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1169 {
1170     gnutls_free(datum->data);
1171     datum->data = NULL;
1172     datum->size = 0;
1173 }
1174 
1175 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1176 
1177 // \return Standard Pacemaker return code
1178 static int
1179 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1180 {
1181     FILE *stream = NULL;
1182     size_t buf_len = KEY_READ_LEN;
1183 
1184     if ((location == NULL) || (key == NULL)) {
1185         return EINVAL;
1186     }
1187 
1188     stream = fopen(location, "r");
1189     if (stream == NULL) {
1190         return errno;
1191     }
1192 
1193     key->data = gnutls_malloc(buf_len);
1194     key->size = 0;
1195     while (!feof(stream)) {
1196         int next = fgetc(stream);
1197 
1198         if (next == EOF) {
1199             if (!feof(stream)) {
1200                 crm_warn("Pacemaker Remote key read was partially successful "
1201                          "(copy in memory may be corrupted)");
1202             }
1203             break;
1204         }
1205         if (key->size == buf_len) {
1206             buf_len = key->size + KEY_READ_LEN;
1207             key->data = gnutls_realloc(key->data, buf_len);
1208             pcmk__assert(key->data);
1209         }
1210         key->data[key->size++] = (unsigned char) next;
1211     }
1212     fclose(stream);
1213 
1214     if (key->size == 0) {
1215         clear_gnutls_datum(key);
1216         return ENOKEY;
1217     }
1218     return pcmk_rc_ok;
1219 }
1220 
1221 // Cache the most recently used Pacemaker Remote authentication key
1222 
1223 struct key_cache_s {
1224     time_t updated;         // When cached key was read (valid for 1 minute)
1225     const char *location;   // Where cached key was read from
1226     gnutls_datum_t key;     // Cached key
1227 };
1228 
1229 static bool
1230 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1231 {
1232     return key_cache->updated != 0;
1233 }
1234 
1235 static bool
1236 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1237 {
1238     return (time(NULL) - key_cache->updated) >= 60;
1239 }
1240 
1241 static void
1242 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1243 {
1244     clear_gnutls_datum(&(key_cache->key));
1245     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1246         key_cache->updated = 0;
1247         key_cache->location = NULL;
1248         crm_debug("Cleared Pacemaker Remote key cache");
1249     }
1250 }
1251 
1252 static void
1253 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1254 {
1255     copy_gnutls_datum(key, &(key_cache->key));
1256     crm_debug("Using cached Pacemaker Remote key from %s",
1257               pcmk__s(key_cache->location, "unknown location"));
1258 }
1259 
1260 static void
1261 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1262           const char *location)
1263 {
1264     key_cache->updated = time(NULL);
1265     key_cache->location = location;
1266     copy_gnutls_datum(&(key_cache->key), key);
1267     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1268               pcmk__s(location, "unknown location"));
1269 }
1270 
1271 /*!
1272  * \internal
1273  * \brief Get Pacemaker Remote authentication key from file or cache
1274  *
1275  * \param[in]  location         Path to key file to try (this memory must
1276  *                              persist across all calls of this function)
1277  * \param[out] key              Key from location or cache
1278  *
1279  * \return Standard Pacemaker return code
1280  */
1281 static int
1282 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1283 {
1284     static struct key_cache_s key_cache = { 0, };
1285     int rc = pcmk_rc_ok;
1286 
1287     if ((location == NULL) || (key == NULL)) {
1288         return EINVAL;
1289     }
1290 
1291     if (key_is_cached(&key_cache)) {
1292         if (key_cache_expired(&key_cache)) {
1293             clear_key_cache(&key_cache);
1294         } else {
1295             get_cached_key(&key_cache, key);
1296             return pcmk_rc_ok;
1297         }
1298     }
1299 
1300     rc = read_gnutls_key(location, key);
1301     if (rc != pcmk_rc_ok) {
1302         return rc;
1303     }
1304     cache_key(&key_cache, key, location);
1305     return pcmk_rc_ok;
1306 }
1307 
1308 /*!
1309  * \internal
1310  * \brief Initialize the Pacemaker Remote authentication key
1311  *
1312  * Try loading the Pacemaker Remote authentication key from cache if available,
1313  * otherwise from these locations, in order of preference:
1314  *
1315  * - The value of the PCMK_authkey_location environment variable, if set
1316  * - The Pacemaker default key file location
1317  *
1318  * \param[out] key  Where to store key
1319  *
1320  * \return Standard Pacemaker return code
1321  */
1322 int
1323 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1324 {
1325     static const char *env_location = NULL;
1326     static bool need_env = true;
1327 
1328     int rc = pcmk_rc_ok;
1329 
1330     if (need_env) {
1331         env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
1332         need_env = false;
1333     }
1334 
1335     // Try location in environment variable, if set
1336     if (env_location != NULL) {
1337         rc = get_remote_key(env_location, key);
1338         if (rc == pcmk_rc_ok) {
1339             return pcmk_rc_ok;
1340         }
1341 
1342         crm_warn("Could not read Pacemaker Remote key from %s: %s",
1343                  env_location, pcmk_rc_str(rc));
1344         return ENOKEY;
1345     }
1346 
1347     // Try default location, if environment wasn't explicitly set to it
1348     rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1349     if (rc == pcmk_rc_ok) {
1350         return pcmk_rc_ok;
1351     }
1352 
1353     crm_warn("Could not read Pacemaker Remote key from default location %s: %s",
1354              DEFAULT_REMOTE_KEY_LOCATION, pcmk_rc_str(rc));
1355     return ENOKEY;
1356 }
1357 
1358 static void
1359 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1360 {
1361     lrmd_private_t *native = lrmd->lrmd_private;
1362 
1363     if (native->callback) {
1364         lrmd_event_data_t event = { 0, };
1365         event.type = lrmd_event_connect;
1366         event.remote_nodename = native->remote_nodename;
1367         event.connection_rc = rc;
1368         native->callback(&event);
1369     }
1370 }
1371 
1372 static void
1373 tls_handshake_failed(lrmd_t *lrmd, int tls_rc, int rc)
     /*  */
1374 {
1375     lrmd_private_t *native = lrmd->lrmd_private;
1376 
1377     crm_warn("Disconnecting after TLS handshake with "
1378              "Pacemaker Remote server %s:%d failed: %s",
1379              native->server, native->port,
1380              (rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc));
1381     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1382 
1383     gnutls_deinit(native->remote->tls_session);
1384     native->remote->tls_session = NULL;
1385     lrmd_tls_connection_destroy(lrmd);
1386 }
1387 
1388 static void
1389 tls_handshake_succeeded(lrmd_t *lrmd)
     /*  */
1390 {
1391     int rc = pcmk_rc_ok;
1392     lrmd_private_t *native = lrmd->lrmd_private;
1393 
1394     /* Now that the handshake is done, see if any client TLS certificate is
1395      * close to its expiration date and log if so.  If a TLS certificate is not
1396      * in use, this function will just return so we don't need to check for the
1397      * session type here.
1398      */
1399     pcmk__tls_check_cert_expiration(native->remote->tls_session);
1400 
1401     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1402              native->server, native->port);
1403     rc = add_tls_to_mainloop(lrmd, true);
1404 
1405     /* If add_tls_to_mainloop failed, report that right now.  Otherwise, we have
1406      * to wait until we read the async reply to report anything.
1407      */
1408     if (rc != pcmk_rc_ok) {
1409         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1410     }
1411 }
1412 
1413 /*!
1414  * \internal
1415  * \brief Perform a TLS client handshake with a Pacemaker Remote server
1416  *
1417  * \param[in] lrmd  Newly established Pacemaker Remote executor connection
1418  *
1419  * \return Standard Pacemaker return code
1420  */
1421 static int
1422 tls_client_handshake(lrmd_t *lrmd)
     /*  */
1423 {
1424     lrmd_private_t *native = lrmd->lrmd_private;
1425     int tls_rc = GNUTLS_E_SUCCESS;
1426     int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT,
1427                                         &tls_rc);
1428 
1429     if (rc != pcmk_rc_ok) {
1430         tls_handshake_failed(lrmd, tls_rc, rc);
1431     }
1432 
1433     return rc;
1434 }
1435 
1436 /*!
1437  * \internal
1438  * \brief Add trigger and file descriptor mainloop sources for TLS
1439  *
1440  * \param[in,out] lrmd              API connection with established TLS session
1441  * \param[in]     do_api_handshake  Whether to perform executor handshake
1442  *
1443  * \return Standard Pacemaker return code
1444  */
1445 static int
1446 add_tls_to_mainloop(lrmd_t *lrmd, bool do_api_handshake)
     /*  */
1447 {
1448     lrmd_private_t *native = lrmd->lrmd_private;
1449     int rc = pcmk_rc_ok;
1450 
1451     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1452                                    native->server, native->port);
1453 
1454     struct mainloop_fd_callbacks tls_fd_callbacks = {
1455         .dispatch = lrmd_tls_dispatch,
1456         .destroy = lrmd_tls_connection_destroy,
1457     };
1458 
1459     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1460                                                   process_pending_notifies, lrmd);
1461     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1462                                      &tls_fd_callbacks);
1463 
1464     /* Async connections lose the client name provided by the API caller, so we
1465      * have to use our generated name here to perform the executor handshake.
1466      *
1467      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1468      * that name in this function instead of generating one anyway.
1469      */
1470     if (do_api_handshake) {
1471         rc = lrmd_handshake_async(lrmd, name);
1472     }
1473     free(name);
1474     return rc;
1475 }
1476 
1477 struct handshake_data_s {
1478     lrmd_t *lrmd;
1479     time_t start_time;
1480     int timeout_sec;
1481 };
1482 
1483 static gboolean
1484 try_handshake_cb(gpointer user_data)
     /*  */
1485 {
1486     struct handshake_data_s *hs = user_data;
1487     lrmd_t *lrmd = hs->lrmd;
1488     lrmd_private_t *native = lrmd->lrmd_private;
1489     pcmk__remote_t *remote = native->remote;
1490 
1491     int rc = pcmk_rc_ok;
1492     int tls_rc = GNUTLS_E_SUCCESS;
1493 
1494     if (time(NULL) >= hs->start_time + hs->timeout_sec) {
1495         rc = ETIME;
1496 
1497         tls_handshake_failed(lrmd, GNUTLS_E_TIMEDOUT, rc);
1498         free(hs);
1499         return 0;
1500     }
1501 
1502     rc = pcmk__tls_client_try_handshake(remote, &tls_rc);
1503 
1504     if (rc == pcmk_rc_ok) {
1505         tls_handshake_succeeded(lrmd);
1506         free(hs);
1507         return 0;
1508     } else if (rc == EAGAIN) {
1509         mainloop_set_trigger(native->handshake_trigger);
1510         return 1;
1511     } else {
1512         rc = EKEYREJECTED;
1513         tls_handshake_failed(lrmd, tls_rc, rc);
1514         free(hs);
1515         return 0;
1516     }
1517 }
1518 
1519 static void
1520 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1521 {
1522     lrmd_t *lrmd = userdata;
1523     lrmd_private_t *native = lrmd->lrmd_private;
1524     int tls_rc = GNUTLS_E_SUCCESS;
1525     bool use_cert = pcmk__x509_enabled();
1526 
1527     native->async_timer = 0;
1528 
1529     if (rc != pcmk_rc_ok) {
1530         lrmd_tls_connection_destroy(lrmd);
1531         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1532                  QB_XS " rc=%d",
1533                  native->server, native->port, pcmk_rc_str(rc), rc);
1534         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1535         return;
1536     }
1537 
1538     /* The TCP connection was successful, so establish the TLS connection. */
1539 
1540     native->sock = sock;
1541 
1542     if (native->tls == NULL) {
1543         rc = pcmk__init_tls(&native->tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_PSK);
1544 
1545         if (rc != pcmk_rc_ok) {
1546             lrmd_tls_connection_destroy(lrmd);
1547             report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1548             return;
1549         }
1550     }
1551 
1552     if (!use_cert) {
1553         gnutls_datum_t psk_key = { NULL, 0 };
1554 
1555         rc = lrmd__init_remote_key(&psk_key);
1556         if (rc != pcmk_rc_ok) {
1557             crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1558                      QB_XS " rc=%d",
1559                      native->server, native->port, pcmk_rc_str(rc), rc);
1560             lrmd_tls_connection_destroy(lrmd);
1561             report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1562             return;
1563         }
1564 
1565         pcmk__tls_add_psk_key(native->tls, &psk_key);
1566         gnutls_free(psk_key.data);
1567     }
1568 
1569     native->remote->tls_session = pcmk__new_tls_session(native->tls, sock);
1570     if (native->remote->tls_session == NULL) {
1571         lrmd_tls_connection_destroy(lrmd);
1572         report_async_connection_result(lrmd, -EPROTO);
1573         return;
1574     }
1575 
1576     /* If the TLS handshake immediately succeeds or fails, we can handle that
1577      * now without having to deal with mainloops and retries.  Otherwise, add a
1578      * trigger to keep trying until we get a result (or it times out).
1579      */
1580     rc = pcmk__tls_client_try_handshake(native->remote, &tls_rc);
1581     if (rc == EAGAIN) {
1582         struct handshake_data_s *hs = NULL;
1583 
1584         if (native->handshake_trigger != NULL) {
1585             return;
1586         }
1587 
1588         hs = pcmk__assert_alloc(1, sizeof(struct handshake_data_s));
1589         hs->lrmd = lrmd;
1590         hs->start_time = time(NULL);
1591         hs->timeout_sec = TLS_HANDSHAKE_TIMEOUT;
1592 
1593         native->handshake_trigger = mainloop_add_trigger(G_PRIORITY_LOW, try_handshake_cb, hs);
1594         mainloop_set_trigger(native->handshake_trigger);
1595 
1596     } else if (rc == pcmk_rc_ok) {
1597         tls_handshake_succeeded(lrmd);
1598 
1599     } else {
1600         tls_handshake_failed(lrmd, tls_rc, rc);
1601     }
1602 }
1603 
1604 static int
1605 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1606 {
1607     int rc = pcmk_rc_ok;
1608     int timer_id = 0;
1609     lrmd_private_t *native = lrmd->lrmd_private;
1610 
1611     native->sock = -1;
1612     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1613                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1614     if (rc != pcmk_rc_ok) {
1615         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1616                  QB_XS " rc=%d",
1617                  native->server, native->port, pcmk_rc_str(rc), rc);
1618         return rc;
1619     }
1620     native->async_timer = timer_id;
1621     return rc;
1622 }
1623 
1624 static int
1625 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1626 {
1627     int rc = pcmk_rc_ok;
1628     bool use_cert = pcmk__x509_enabled();
1629     lrmd_private_t *native = lrmd->lrmd_private;
1630 
1631     native->sock = -1;
1632     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1633                               &(native->sock), NULL, NULL);
1634     if (rc != pcmk_rc_ok) {
1635         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1636                  QB_XS " rc=%d",
1637                  native->server, native->port, pcmk_rc_str(rc), rc);
1638         lrmd_tls_connection_destroy(lrmd);
1639         return ENOTCONN;
1640     }
1641 
1642     if (native->tls == NULL) {
1643         rc = pcmk__init_tls(&native->tls, false, use_cert ? GNUTLS_CRD_CERTIFICATE : GNUTLS_CRD_PSK);
1644 
1645         if (rc != pcmk_rc_ok) {
1646             lrmd_tls_connection_destroy(lrmd);
1647             return rc;
1648         }
1649     }
1650 
1651     if (!use_cert) {
1652         gnutls_datum_t psk_key = { NULL, 0 };
1653 
1654         rc = lrmd__init_remote_key(&psk_key);
1655         if (rc != pcmk_rc_ok) {
1656             lrmd_tls_connection_destroy(lrmd);
1657             return rc;
1658         }
1659 
1660         pcmk__tls_add_psk_key(native->tls, &psk_key);
1661         gnutls_free(psk_key.data);
1662     }
1663 
1664     native->remote->tls_session = pcmk__new_tls_session(native->tls, native->sock);
1665     if (native->remote->tls_session == NULL) {
1666         lrmd_tls_connection_destroy(lrmd);
1667         return EPROTO;
1668     }
1669 
1670     if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
1671         return EKEYREJECTED;
1672     }
1673 
1674     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1675              native->port);
1676 
1677     if (fd) {
1678         *fd = native->sock;
1679     } else {
1680         rc = add_tls_to_mainloop(lrmd, false);
1681     }
1682     return rc;
1683 }
1684 
1685 static int
1686 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1687 {
1688     int rc = -ENOTCONN;
1689     lrmd_private_t *native = lrmd->lrmd_private;
1690 
1691     switch (native->type) {
1692         case pcmk__client_ipc:
1693             rc = lrmd_ipc_connect(lrmd, fd);
1694             break;
1695         case pcmk__client_tls:
1696             rc = lrmd_tls_connect(lrmd, fd);
1697             rc = pcmk_rc2legacy(rc);
1698             break;
1699         default:
1700             crm_err("Unsupported executor connection type (bug?): %d",
1701                     native->type);
1702             rc = -EPROTONOSUPPORT;
1703     }
1704 
1705     if (rc == pcmk_ok) {
1706         rc = lrmd_handshake(lrmd, name);
1707         rc = pcmk_rc2legacy(rc);
1708     }
1709 
1710     return rc;
1711 }
1712 
1713 static int
1714 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1715 {
1716     int rc = pcmk_ok;
1717     lrmd_private_t *native = lrmd->lrmd_private;
1718 
1719     CRM_CHECK(native && native->callback, return -EINVAL);
1720 
1721     switch (native->type) {
1722         case pcmk__client_ipc:
1723             /* fake async connection with ipc.  it should be fast
1724              * enough that we gain very little from async */
1725             rc = lrmd_api_connect(lrmd, name, NULL);
1726             if (!rc) {
1727                 report_async_connection_result(lrmd, rc);
1728             }
1729             break;
1730         case pcmk__client_tls:
1731             rc = lrmd_tls_connect_async(lrmd, timeout);
1732             rc = pcmk_rc2legacy(rc);
1733             break;
1734         default:
1735             crm_err("Unsupported executor connection type (bug?): %d",
1736                     native->type);
1737             rc = -EPROTONOSUPPORT;
1738     }
1739 
1740     return rc;
1741 }
1742 
1743 static void
1744 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1745 {
1746     lrmd_private_t *native = lrmd->lrmd_private;
1747 
1748     if (native->source != NULL) {
1749         /* Attached to mainloop */
1750         mainloop_del_ipc_client(native->source);
1751         native->source = NULL;
1752         native->ipc = NULL;
1753 
1754     } else if (native->ipc) {
1755         /* Not attached to mainloop */
1756         crm_ipc_t *ipc = native->ipc;
1757 
1758         native->ipc = NULL;
1759         crm_ipc_close(ipc);
1760         crm_ipc_destroy(ipc);
1761     }
1762 }
1763 
1764 static void
1765 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1766 {
1767     lrmd_private_t *native = lrmd->lrmd_private;
1768 
1769     if (native->remote->tls_session) {
1770         gnutls_bye(native->remote->tls_session, GNUTLS_SHUT_RDWR);
1771         gnutls_deinit(native->remote->tls_session);
1772         native->remote->tls_session = NULL;
1773     }
1774 
1775     if (native->async_timer) {
1776         g_source_remove(native->async_timer);
1777         native->async_timer = 0;
1778     }
1779 
1780     if (native->source != NULL) {
1781         /* Attached to mainloop */
1782         mainloop_del_ipc_client(native->source);
1783         native->source = NULL;
1784 
1785     } else if (native->sock) {
1786         close(native->sock);
1787         native->sock = 0;
1788     }
1789 
1790     if (native->pending_notify) {
1791         g_list_free_full(native->pending_notify, lrmd_free_xml);
1792         native->pending_notify = NULL;
1793     }
1794 }
1795 
1796 static int
1797 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1798 {
1799     lrmd_private_t *native = lrmd->lrmd_private;
1800     int rc = pcmk_ok;
1801 
1802     switch (native->type) {
1803         case pcmk__client_ipc:
1804             crm_debug("Disconnecting from local executor");
1805             lrmd_ipc_disconnect(lrmd);
1806             break;
1807         case pcmk__client_tls:
1808             crm_debug("Disconnecting from remote executor on %s",
1809                       native->remote_nodename);
1810             lrmd_tls_disconnect(lrmd);
1811             break;
1812         default:
1813             crm_err("Unsupported executor connection type (bug?): %d",
1814                     native->type);
1815             rc = -EPROTONOSUPPORT;
1816     }
1817 
1818     free(native->token);
1819     native->token = NULL;
1820 
1821     free(native->peer_version);
1822     native->peer_version = NULL;
1823     return rc;
1824 }
1825 
1826 static int
1827 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1828                       const char *rsc_id,
1829                       const char *class,
1830                       const char *provider, const char *type, enum lrmd_call_options options)
1831 {
1832     int rc = pcmk_ok;
1833     xmlNode *data = NULL;
1834 
1835     if (!class || !type || !rsc_id) {
1836         return -EINVAL;
1837     }
1838     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1839         && (provider == NULL)) {
1840         return -EINVAL;
1841     }
1842 
1843     data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1844 
1845     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1846     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1847     crm_xml_add(data, PCMK__XA_LRMD_CLASS, class);
1848     crm_xml_add(data, PCMK__XA_LRMD_PROVIDER, provider);
1849     crm_xml_add(data, PCMK__XA_LRMD_TYPE, type);
1850     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, true);
1851     pcmk__xml_free(data);
1852 
1853     return rc;
1854 }
1855 
1856 static int
1857 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1858 {
1859     int rc = pcmk_ok;
1860     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1861 
1862     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1863     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1864     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, true);
1865     pcmk__xml_free(data);
1866 
1867     return rc;
1868 }
1869 
1870 lrmd_rsc_info_t *
1871 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1872                   const char *provider, const char *type)
1873 {
1874     lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t));
1875 
1876     rsc_info->id = pcmk__str_copy(rsc_id);
1877     rsc_info->standard = pcmk__str_copy(standard);
1878     rsc_info->provider = pcmk__str_copy(provider);
1879     rsc_info->type = pcmk__str_copy(type);
1880     return rsc_info;
1881 }
1882 
1883 lrmd_rsc_info_t *
1884 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1885 {
1886     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1887                              rsc_info->provider, rsc_info->type);
1888 }
1889 
1890 void
1891 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1892 {
1893     if (!rsc_info) {
1894         return;
1895     }
1896     free(rsc_info->id);
1897     free(rsc_info->type);
1898     free(rsc_info->standard);
1899     free(rsc_info->provider);
1900     free(rsc_info);
1901 }
1902 
1903 static lrmd_rsc_info_t *
1904 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1905 {
1906     lrmd_rsc_info_t *rsc_info = NULL;
1907     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1908     xmlNode *output = NULL;
1909     const char *class = NULL;
1910     const char *provider = NULL;
1911     const char *type = NULL;
1912 
1913     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1914     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1915     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, true);
1916     pcmk__xml_free(data);
1917 
1918     if (!output) {
1919         return NULL;
1920     }
1921 
1922     class = crm_element_value(output, PCMK__XA_LRMD_CLASS);
1923     provider = crm_element_value(output, PCMK__XA_LRMD_PROVIDER);
1924     type = crm_element_value(output, PCMK__XA_LRMD_TYPE);
1925 
1926     if (!class || !type) {
1927         pcmk__xml_free(output);
1928         return NULL;
1929     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1930                && !provider) {
1931         pcmk__xml_free(output);
1932         return NULL;
1933     }
1934 
1935     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1936     pcmk__xml_free(output);
1937     return rsc_info;
1938 }
1939 
1940 void
1941 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1942 {
1943     if (op_info) {
1944         free(op_info->rsc_id);
1945         free(op_info->action);
1946         free(op_info->interval_ms_s);
1947         free(op_info->timeout_ms_s);
1948         free(op_info);
1949     }
1950 }
1951 
1952 static int
1953 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1954                            enum lrmd_call_options options, GList **output)
1955 {
1956     xmlNode *data = NULL;
1957     xmlNode *output_xml = NULL;
1958     int rc = pcmk_ok;
1959 
1960     if (output == NULL) {
1961         return -EINVAL;
1962     }
1963     *output = NULL;
1964 
1965     // Send request
1966     if (rsc_id) {
1967         data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1968         crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1969         crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1970     }
1971     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1972                            timeout_ms, options, true);
1973     if (data) {
1974         pcmk__xml_free(data);
1975     }
1976 
1977     // Process reply
1978     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1979         return rc;
1980     }
1981     for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml,
1982                                                        PCMK__XE_LRMD_RSC, NULL,
1983                                                        NULL);
1984          (rsc_xml != NULL) && (rc == pcmk_ok);
1985          rsc_xml = pcmk__xe_next(rsc_xml, PCMK__XE_LRMD_RSC)) {
1986 
1987         rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
1988         if (rsc_id == NULL) {
1989             crm_err("Could not parse recurring operation information from executor");
1990             continue;
1991         }
1992         for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml,
1993                                                           PCMK__XE_LRMD_RSC_OP,
1994                                                           NULL, NULL);
1995              op_xml != NULL;
1996              op_xml = pcmk__xe_next(op_xml, PCMK__XE_LRMD_RSC_OP)) {
1997 
1998             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1999 
2000             if (op_info == NULL) {
2001                 rc = -ENOMEM;
2002                 break;
2003             }
2004             op_info->rsc_id = strdup(rsc_id);
2005             op_info->action = crm_element_value_copy(op_xml,
2006                                                      PCMK__XA_LRMD_RSC_ACTION);
2007             op_info->interval_ms_s =
2008                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
2009             op_info->timeout_ms_s =
2010                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_TIMEOUT);
2011             *output = g_list_prepend(*output, op_info);
2012         }
2013     }
2014     pcmk__xml_free(output_xml);
2015     return rc;
2016 }
2017 
2018 
2019 static void
2020 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
2021 {
2022     lrmd_private_t *native = lrmd->lrmd_private;
2023 
2024     native->callback = callback;
2025 }
2026 
2027 void
2028 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
2029 {
2030     lrmd_private_t *native = lrmd->lrmd_private;
2031 
2032     native->proxy_callback = callback;
2033     native->proxy_callback_userdata = userdata;
2034 }
2035 
2036 void
2037 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
2038 {
2039     lrmd_private_t *native = lrmd->lrmd_private;
2040 
2041     if (native->proxy_callback) {
2042         crm_log_xml_trace(msg, "PROXY_INBOUND");
2043         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
2044     }
2045 }
2046 
2047 int
2048 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
2049 {
2050     if (lrmd == NULL) {
2051         return -ENOTCONN;
2052     }
2053     crm_xml_add(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);
2054 
2055     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
2056     return lrmd_send_xml_no_reply(lrmd, msg);
2057 }
2058 
2059 static int
2060 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
2061 {
2062     int rc = pcmk_ok;
2063     stonith_t *stonith_api = stonith_api_new();
2064 
2065     if (stonith_api == NULL) {
2066         crm_err("Could not get fence agent meta-data: API memory allocation failed");
2067         return -ENOMEM;
2068     }
2069 
2070     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
2071                                      provider, output, 0);
2072     if ((rc == pcmk_ok) && (*output == NULL)) {
2073         rc = -EIO;
2074     }
2075     stonith_api->cmds->free(stonith_api);
2076     return rc;
2077 }
2078 
2079 static int
2080 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
2081                       const char *type, char **output,
2082                       enum lrmd_call_options options)
2083 {
2084     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
2085                                            output, options, NULL);
2086 }
2087 
2088 static int
2089 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
2090                              const char *provider, const char *type,
2091                              char **output, enum lrmd_call_options options,
2092                              lrmd_key_value_t *params)
2093 {
2094     svc_action_t *action = NULL;
2095     GHashTable *params_table = NULL;
2096 
2097     if (!standard || !type) {
2098         lrmd_key_value_freeall(params);
2099         return -EINVAL;
2100     }
2101 
2102     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2103         lrmd_key_value_freeall(params);
2104         return stonith_get_metadata(provider, type, output);
2105     }
2106 
2107     params_table = pcmk__strkey_table(free, free);
2108     for (const lrmd_key_value_t *param = params; param; param = param->next) {
2109         pcmk__insert_dup(params_table, param->key, param->value);
2110     }
2111     action = services__create_resource_action(type, standard, provider, type,
2112                                               PCMK_ACTION_META_DATA, 0,
2113                                               PCMK_DEFAULT_ACTION_TIMEOUT_MS,
2114                                               params_table, 0);
2115     lrmd_key_value_freeall(params);
2116 
2117     if (action == NULL) {
2118         return -ENOMEM;
2119     }
2120     if (action->rc != PCMK_OCF_UNKNOWN) {
2121         services_action_free(action);
2122         return -EINVAL;
2123     }
2124 
2125     if (!services_action_sync(action)) {
2126         crm_err("Failed to retrieve meta-data for %s:%s:%s",
2127                 standard, provider, type);
2128         services_action_free(action);
2129         return -EIO;
2130     }
2131 
2132     if (!action->stdout_data) {
2133         crm_err("Failed to receive meta-data for %s:%s:%s",
2134                 standard, provider, type);
2135         services_action_free(action);
2136         return -EIO;
2137     }
2138 
2139     *output = strdup(action->stdout_data);
2140     services_action_free(action);
2141 
2142     return pcmk_ok;
2143 }
2144 
2145 static int
2146 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2147               const char *userdata, guint interval_ms,
2148               int timeout,      /* ms */
2149               int start_delay,  /* ms */
2150               enum lrmd_call_options options, lrmd_key_value_t * params)
2151 {
2152     int rc = pcmk_ok;
2153     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2154     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2155     lrmd_key_value_t *tmp = NULL;
2156 
2157     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2158     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2159     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2160     crm_xml_add(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
2161     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2162     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2163     crm_xml_add_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);
2164 
2165     for (tmp = params; tmp; tmp = tmp->next) {
2166         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2167     }
2168 
2169     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, true);
2170     pcmk__xml_free(data);
2171 
2172     lrmd_key_value_freeall(params);
2173     return rc;
2174 }
2175 
2176 /* timeout is in ms */
2177 static int
2178 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2179                     int timeout, lrmd_key_value_t *params)
2180 {
2181     int rc = pcmk_ok;
2182     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT);
2183     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2184     lrmd_key_value_t *tmp = NULL;
2185 
2186     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2187     crm_xml_add(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
2188     crm_xml_add(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
2189     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2190 
2191     for (tmp = params; tmp; tmp = tmp->next) {
2192         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2193     }
2194 
2195     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2196                            lrmd_opt_notify_orig_only, true);
2197     pcmk__xml_free(data);
2198 
2199     lrmd_key_value_freeall(params);
2200     return rc;
2201 }
2202 
2203 static int
2204 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2205                 guint interval_ms)
2206 {
2207     int rc = pcmk_ok;
2208     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2209 
2210     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2211     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2212     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2213     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2214     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, true);
2215     pcmk__xml_free(data);
2216     return rc;
2217 }
2218 
2219 static int
2220 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2221 {
2222     int rc = 0;
2223     stonith_t *stonith_api = stonith_api_new();
2224     stonith_key_value_t *stonith_resources = NULL;
2225     stonith_key_value_t *dIter = NULL;
2226 
2227     if (stonith_api == NULL) {
2228         crm_err("Could not list fence agents: API memory allocation failed");
2229         return -ENOMEM;
2230     }
2231     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2232                                    &stonith_resources, 0);
2233     stonith_api->cmds->free(stonith_api);
2234 
2235     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2236         rc++;
2237         if (resources) {
2238             *resources = lrmd_list_add(*resources, dIter->value);
2239         }
2240     }
2241 
2242     stonith_key_value_freeall(stonith_resources, 1, 0);
2243     return rc;
2244 }
2245 
2246 static int
2247 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2248                      const char *provider)
2249 {
2250     int rc = 0;
2251     int stonith_count = 0; // Initially, whether to include stonith devices
2252 
2253     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2254         stonith_count = 1;
2255 
2256     } else {
2257         GList *gIter = NULL;
2258         GList *agents = resources_list_agents(class, provider);
2259 
2260         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2261             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2262             rc++;
2263         }
2264         g_list_free_full(agents, free);
2265 
2266         if (!class) {
2267             stonith_count = 1;
2268         }
2269     }
2270 
2271     if (stonith_count) {
2272         // Now, if stonith devices are included, how many there are
2273         stonith_count = list_stonith_agents(resources);
2274         if (stonith_count > 0) {
2275             rc += stonith_count;
2276         }
2277     }
2278     if (rc == 0) {
2279         crm_notice("No agents found for class %s", class);
2280         rc = -EPROTONOSUPPORT;
2281     }
2282     return rc;
2283 }
2284 
2285 static bool
2286 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2287 {
2288     bool found = false;
2289     GList *agents = NULL;
2290     GList *gIter2 = NULL;
2291 
2292     agents = resources_list_agents(class, provider);
2293     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2294         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2295             found = true;
2296         }
2297     }
2298     g_list_free_full(agents, free);
2299     return found;
2300 }
2301 
2302 static int
2303 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2304 {
2305     int rc = pcmk_ok;
2306     char *provider = NULL;
2307     GList *ocf_providers = NULL;
2308     GList *gIter = NULL;
2309 
2310     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2311 
2312     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2313         provider = gIter->data;
2314         if (!agent || does_provider_have_agent(agent, provider,
2315                                                PCMK_RESOURCE_CLASS_OCF)) {
2316             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2317             rc++;
2318         }
2319     }
2320 
2321     g_list_free_full(ocf_providers, free);
2322     return rc;
2323 }
2324 
2325 static int
2326 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2327 {
2328     int rc = 0;
2329     GList *standards = NULL;
2330     GList *gIter = NULL;
2331 
2332     standards = resources_list_standards();
2333 
2334     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2335         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2336         rc++;
2337     }
2338 
2339     if (list_stonith_agents(NULL) > 0) {
2340         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2341         rc++;
2342     }
2343 
2344     g_list_free_full(standards, free);
2345     return rc;
2346 }
2347 
2348 /*!
2349  * \internal
2350  * \brief Create an executor API object
2351  *
2352  * \param[out] api       Will be set to newly created API object (it is the
2353  *                       caller's responsibility to free this value with
2354  *                       lrmd_api_delete() if this function succeeds)
2355  * \param[in]  nodename  If the object will be used for a remote connection,
2356  *                       the node name to use in cluster for remote executor
2357  * \param[in]  server    If the object will be used for a remote connection,
2358  *                       the resolvable host name to connect to
2359  * \param[in]  port      If the object will be used for a remote connection,
2360  *                       port number on \p server to connect to
2361  *
2362  * \return Standard Pacemaker return code
2363  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2364  *       value will be used for both. If the caller leaves both NULL, an API
2365  *       object will be created for a local executor connection.
2366  */
2367 int
2368 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2369 {
2370     lrmd_private_t *pvt = NULL;
2371 
2372     if (api == NULL) {
2373         return EINVAL;
2374     }
2375     *api = NULL;
2376 
2377     // Allocate all memory needed
2378 
2379     *api = calloc(1, sizeof(lrmd_t));
2380     if (*api == NULL) {
2381         return ENOMEM;
2382     }
2383 
2384     pvt = calloc(1, sizeof(lrmd_private_t));
2385     if (pvt == NULL) {
2386         lrmd_api_delete(*api);
2387         *api = NULL;
2388         return ENOMEM;
2389     }
2390     (*api)->lrmd_private = pvt;
2391 
2392     // @TODO Do we need to do this for local connections?
2393     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2394 
2395     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2396 
2397     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2398         lrmd_api_delete(*api);
2399         *api = NULL;
2400         return ENOMEM;
2401     }
2402 
2403     // Set methods
2404     (*api)->cmds->connect = lrmd_api_connect;
2405     (*api)->cmds->connect_async = lrmd_api_connect_async;
2406     (*api)->cmds->is_connected = lrmd_api_is_connected;
2407     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2408     (*api)->cmds->disconnect = lrmd_api_disconnect;
2409     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2410     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2411     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2412     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2413     (*api)->cmds->set_callback = lrmd_api_set_callback;
2414     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2415     (*api)->cmds->exec = lrmd_api_exec;
2416     (*api)->cmds->cancel = lrmd_api_cancel;
2417     (*api)->cmds->list_agents = lrmd_api_list_agents;
2418     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2419     (*api)->cmds->list_standards = lrmd_api_list_standards;
2420     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2421     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2422 
2423     if ((nodename == NULL) && (server == NULL)) {
2424         pvt->type = pcmk__client_ipc;
2425     } else {
2426         if (nodename == NULL) {
2427             nodename = server;
2428         } else if (server == NULL) {
2429             server = nodename;
2430         }
2431         pvt->type = pcmk__client_tls;
2432         pvt->remote_nodename = strdup(nodename);
2433         pvt->server = strdup(server);
2434         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2435             lrmd_api_delete(*api);
2436             *api = NULL;
2437             return ENOMEM;
2438         }
2439         pvt->port = port;
2440         if (pvt->port == 0) {
2441             pvt->port = crm_default_remote_port();
2442         }
2443     }
2444     return pcmk_rc_ok;
2445 }
2446 
2447 lrmd_t *
2448 lrmd_api_new(void)
     /*  */
2449 {
2450     lrmd_t *api = NULL;
2451 
2452     pcmk__assert(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2453     return api;
2454 }
2455 
2456 lrmd_t *
2457 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2458 {
2459     lrmd_t *api = NULL;
2460 
2461     pcmk__assert(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2462     return api;
2463 }
2464 
2465 void
2466 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2467 {
2468     if (lrmd == NULL) {
2469         return;
2470     }
2471     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2472         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2473             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2474         }
2475         free(lrmd->cmds);
2476     }
2477     if (lrmd->lrmd_private != NULL) {
2478         lrmd_private_t *native = lrmd->lrmd_private;
2479 
2480         free(native->server);
2481         free(native->remote_nodename);
2482         free(native->remote);
2483         free(native->token);
2484         free(native->peer_version);
2485         free(lrmd->lrmd_private);
2486     }
2487     free(lrmd);
2488 }
2489 
2490 struct metadata_cb {
2491      void (*callback)(int pid, const pcmk__action_result_t *result,
2492                       void *user_data);
2493      void *user_data;
2494 };
2495 
2496 /*!
2497  * \internal
2498  * \brief Process asynchronous metadata completion
2499  *
2500  * \param[in,out] action  Metadata action that completed
2501  */
2502 static void
2503 metadata_complete(svc_action_t *action)
     /*  */
2504 {
2505     struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2506     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2507 
2508     pcmk__set_result(&result, action->rc, action->status,
2509                      services__exit_reason(action));
2510     pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2511 
2512     metadata_cb->callback(0, &result, metadata_cb->user_data);
2513     result.action_stdout = NULL; // Prevent free, because action owns it
2514     result.action_stderr = NULL; // Prevent free, because action owns it
2515     pcmk__reset_result(&result);
2516     free(metadata_cb);
2517 }
2518 
2519 /*!
2520  * \internal
2521  * \brief Retrieve agent metadata asynchronously
2522  *
2523  * \param[in]     rsc        Resource agent specification
2524  * \param[in]     callback   Function to call with result (this will always be
2525  *                           called, whether by this function directly or later
2526  *                           via the main loop, and on success the metadata will
2527  *                           be in its result argument's action_stdout)
2528  * \param[in,out] user_data  User data to pass to callback
2529  *
2530  * \return Standard Pacemaker return code
2531  * \note This function is not a lrmd_api_operations_t method because it does not
2532  *       need an lrmd_t object and does not go through the executor, but
2533  *       executes the agent directly.
2534  */
2535 int
2536 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
     /*  */
2537                      void (*callback)(int pid,
2538                                       const pcmk__action_result_t *result,
2539                                       void *user_data),
2540                      void *user_data)
2541 {
2542     svc_action_t *action = NULL;
2543     struct metadata_cb *metadata_cb = NULL;
2544     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2545 
2546     CRM_CHECK(callback != NULL, return EINVAL);
2547 
2548     if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2549         pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2550                          PCMK_EXEC_ERROR_FATAL,
2551                          "Invalid resource specification");
2552         callback(0, &result, user_data);
2553         pcmk__reset_result(&result);
2554         return EINVAL;
2555     }
2556 
2557     if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2558         return stonith__metadata_async(rsc->type,
2559                                        pcmk__timeout_ms2s(PCMK_DEFAULT_ACTION_TIMEOUT_MS),
2560                                        callback, user_data);
2561     }
2562 
2563     action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2564                                               rsc->standard, rsc->provider,
2565                                               rsc->type,
2566                                               PCMK_ACTION_META_DATA, 0,
2567                                               PCMK_DEFAULT_ACTION_TIMEOUT_MS,
2568                                               NULL, 0);
2569     if (action == NULL) {
2570         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2571                          "Out of memory");
2572         callback(0, &result, user_data);
2573         pcmk__reset_result(&result);
2574         return ENOMEM;
2575     }
2576     if (action->rc != PCMK_OCF_UNKNOWN) {
2577         pcmk__set_result(&result, action->rc, action->status,
2578                          services__exit_reason(action));
2579         callback(0, &result, user_data);
2580         pcmk__reset_result(&result);
2581         services_action_free(action);
2582         return EINVAL;
2583     }
2584 
2585     action->cb_data = calloc(1, sizeof(struct metadata_cb));
2586     if (action->cb_data == NULL) {
2587         services_action_free(action);
2588         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2589                          "Out of memory");
2590         callback(0, &result, user_data);
2591         pcmk__reset_result(&result);
2592         return ENOMEM;
2593     }
2594 
2595     metadata_cb = (struct metadata_cb *) action->cb_data;
2596     metadata_cb->callback = callback;
2597     metadata_cb->user_data = user_data;
2598     if (!services_action_async(action, metadata_complete)) {
2599         services_action_free(action);
2600         return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2601     }
2602 
2603     // The services library has taken responsibility for action
2604     return pcmk_rc_ok;
2605 }
2606 
2607 /*!
2608  * \internal
2609  * \brief Set the result of an executor event
2610  *
2611  * \param[in,out] event        Executor event to set
2612  * \param[in]     rc           OCF exit status of event
2613  * \param[in]     op_status    Executor status of event
2614  * \param[in]     exit_reason  Human-friendly description of event
2615  */
2616 void
2617 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2618                  const char *exit_reason)
2619 {
2620     if (event == NULL) {
2621         return;
2622     }
2623 
2624     event->rc = rc;
2625     event->op_status = op_status;
2626 
2627     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
2628     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2629 }
2630 
2631 /*!
2632  * \internal
2633  * \brief Clear an executor event's exit reason, output, and error output
2634  *
2635  * \param[in,out] event  Executor event to reset
2636  */
2637 void
2638 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2639 {
2640     if (event == NULL) {
2641         return;
2642     }
2643 
2644     free((void *) event->exit_reason);
2645     event->exit_reason = NULL;
2646 
2647     free((void *) event->output);
2648     event->output = NULL;
2649 }
2650 
2651 /*!
2652  * \internal
2653  * \brief Get the uptime of a remote resource connection
2654  *
2655  * When the cluster connects to a remote resource, part of that resource's
2656  * handshake includes the uptime of the remote resource's connection.  This
2657  * uptime is stored in the lrmd_t object.
2658  *
2659  * \return The connection's uptime, or -1 if unknown
2660  */
2661 time_t
2662 lrmd__uptime(lrmd_t *lrmd)
     /*  */
2663 {
2664     lrmd_private_t *native = lrmd->lrmd_private;
2665 
2666     if (native->remote == NULL) {
2667         return -1;
2668     } else {
2669         return native->remote->uptime;
2670     }
2671 }
2672 
2673 const char *
2674 lrmd__node_start_state(lrmd_t *lrmd)
     /*  */
2675 {
2676     lrmd_private_t *native = lrmd->lrmd_private;
2677 
2678     if (native->remote == NULL) {
2679         return NULL;
2680     } else {
2681         return native->remote->start_state;
2682     }
2683 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
