root/lib/lrmd/lrmd_client.c

/* */

DEFINITIONS

1. lrmd_list_add
2. lrmd_list_freeall
3. lrmd_key_value_add
4. lrmd_key_value_freeall
5. lrmd_new_event
6. lrmd_copy_event
7. lrmd_free_event
8. lrmd_dispatch_internal
9. lrmd_ipc_dispatch
10. lrmd_free_xml
11. remote_executor_connected
12. lrmd_tls_dispatch
13. lrmd_poll
14. lrmd_dispatch
15. lrmd_create_op
16. lrmd_ipc_connection_destroy
17. lrmd_tls_connection_destroy
18. lrmd__remote_send_xml
19. read_remote_reply
20. send_remote_message
21. lrmd_tls_send_recv
22. lrmd_send_xml
23. lrmd_send_xml_no_reply
24. lrmd_api_is_connected
25. lrmd_send_command
26. lrmd_api_poke_connection
27. lrmd__validate_remote_settings
28. lrmd_handshake
29. lrmd_ipc_connect
30. copy_gnutls_datum
31. clear_gnutls_datum
32. read_gnutls_key
33. key_is_cached
34. key_cache_expired
35. clear_key_cache
36. get_cached_key
37. cache_key
38. get_remote_key
39. lrmd__init_remote_key
40. lrmd_gnutls_global_init
41. report_async_connection_result
42. lrmd__tls_client_handshake
43. add_tls_to_mainloop
44. lrmd_tcp_connect_cb
45. lrmd_tls_connect_async
46. lrmd_tls_connect
47. lrmd_api_connect
48. lrmd_api_connect_async
49. lrmd_ipc_disconnect
50. lrmd_tls_disconnect
51. lrmd_api_disconnect
52. lrmd_api_register_rsc
53. lrmd_api_unregister_rsc
54. lrmd_new_rsc_info
55. lrmd_copy_rsc_info
56. lrmd_free_rsc_info
57. lrmd_api_get_rsc_info
58. lrmd_free_op_info
59. lrmd_api_get_recurring_ops
60. lrmd_api_set_callback
61. lrmd_internal_set_proxy_callback
62. lrmd_internal_proxy_dispatch
63. lrmd_internal_proxy_send
64. stonith_get_metadata
65. lrmd_api_get_metadata
66. lrmd_api_get_metadata_params
67. lrmd_api_exec
68. lrmd_api_exec_alert
69. lrmd_api_cancel
70. list_stonith_agents
71. lrmd_api_list_agents
72. does_provider_have_agent
73. lrmd_api_list_ocf_providers
74. lrmd_api_list_standards
75. lrmd__new
76. lrmd_api_new
77. lrmd_remote_api_new
78. lrmd_api_delete
79. metadata_complete
80. lrmd__metadata_async
81. lrmd__set_result
82. lrmd__reset_result
83. lrmd__uptime
84. lrmd__node_start_state

   1 /*
   2  * Copyright 2012-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/msg_xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  include <gnutls/gnutls.h>
  42 #endif
  43 
  44 #include <sys/socket.h>
  45 #include <netinet/in.h>
  46 #include <netinet/ip.h>
  47 #include <arpa/inet.h>
  48 #include <netdb.h>
  49 
  50 #define MAX_TLS_RECV_WAIT 10000
  51 
  52 CRM_TRACE_INIT_DATA(lrmd);
  53 
  54 static int lrmd_api_disconnect(lrmd_t * lrmd);
  55 static int lrmd_api_is_connected(lrmd_t * lrmd);
  56 
  57 /* IPC proxy functions */
  58 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  59 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  60 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  61 
  62 #ifdef HAVE_GNUTLS_GNUTLS_H
  63 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  64 gnutls_psk_client_credentials_t psk_cred_s;
  65 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  66 static int global_remote_msg_id = 0;
  67 static void lrmd_tls_connection_destroy(gpointer userdata);
  68 #endif
  69 
  70 typedef struct lrmd_private_s {
  71     uint64_t type;
  72     char *token;
  73     mainloop_io_t *source;
  74 
  75     /* IPC parameters */
  76     crm_ipc_t *ipc;
  77 
  78     pcmk__remote_t *remote;
  79 
  80     /* Extra TLS parameters */
  81     char *remote_nodename;
  82 #ifdef HAVE_GNUTLS_GNUTLS_H
  83     char *server;
  84     int port;
  85     gnutls_psk_client_credentials_t psk_cred_c;
  86 
  87     /* while the async connection is occurring, this is the id
  88      * of the connection timeout timer. */
  89     int async_timer;
  90     int sock;
  91     /* since tls requires a round trip across the network for a
  92      * request/reply, there are times where we just want to be able
  93      * to send a request from the client and not wait around (or even care
  94      * about) what the reply is. */
  95     int expected_late_replies;
  96     GList *pending_notify;
  97     crm_trigger_t *process_notify;
  98 #endif
  99 
 100     lrmd_event_callback callback;
 101 
 102     /* Internal IPC proxy msg passing for remote guests */
 103     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 104     void *proxy_callback_userdata;
 105     char *peer_version;
 106 } lrmd_private_t;
 107 
 108 static lrmd_list_t *
 109 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 110 {
 111     lrmd_list_t *p, *end;
 112 
 113     p = calloc(1, sizeof(lrmd_list_t));
 114     p->val = strdup(value);
 115 
 116     end = head;
 117     while (end && end->next) {
 118         end = end->next;
 119     }
 120 
 121     if (end) {
 122         end->next = p;
 123     } else {
 124         head = p;
 125     }
 126 
 127     return head;
 128 }
 129 
 130 void
 131 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 132 {
 133     lrmd_list_t *p;
 134 
 135     while (head) {
 136         char *val = (char *)head->val;
 137 
 138         p = head->next;
 139         free(val);
 140         free(head);
 141         head = p;
 142     }
 143 }
 144 
 145 lrmd_key_value_t *
 146 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 147 {
 148     lrmd_key_value_t *p, *end;
 149 
 150     p = calloc(1, sizeof(lrmd_key_value_t));
 151     p->key = strdup(key);
 152     p->value = strdup(value);
 153 
 154     end = head;
 155     while (end && end->next) {
 156         end = end->next;
 157     }
 158 
 159     if (end) {
 160         end->next = p;
 161     } else {
 162         head = p;
 163     }
 164 
 165     return head;
 166 }
 167 
 168 void
 169 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 170 {
 171     lrmd_key_value_t *p;
 172 
 173     while (head) {
 174         p = head->next;
 175         free(head->key);
 176         free(head->value);
 177         free(head);
 178         head = p;
 179     }
 180 }
 181 
 182 /*!
 183  * \brief Create a new lrmd_event_data_t object
 184  *
 185  * \param[in] rsc_id       ID of resource involved in event
 186  * \param[in] task         Action name
 187  * \param[in] interval_ms  Action interval
 188  *
 189  * \return Newly allocated and initialized lrmd_event_data_t
 190  * \note This functions asserts on memory errors, so the return value is
 191  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 192  *       result with lrmd_free_event().
 193  */
 194 lrmd_event_data_t *
 195 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 196 {
 197     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 198 
 199     CRM_ASSERT(event != NULL);
 200     pcmk__str_update((char **) &event->rsc_id, rsc_id);
 201     pcmk__str_update((char **) &event->op_type, task);
 202     event->interval_ms = interval_ms;
 203     return event;
 204 }
 205 
 206 lrmd_event_data_t *
 207 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 208 {
 209     lrmd_event_data_t *copy = NULL;
 210 
 211     copy = calloc(1, sizeof(lrmd_event_data_t));
 212 
 213     copy->type = event->type;
 214     pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
 215     pcmk__str_update((char **) &copy->op_type, event->op_type);
 216     pcmk__str_update((char **) &copy->user_data, event->user_data);
 217     copy->call_id = event->call_id;
 218     copy->timeout = event->timeout;
 219     copy->interval_ms = event->interval_ms;
 220     copy->start_delay = event->start_delay;
 221     copy->rsc_deleted = event->rsc_deleted;
 222     copy->rc = event->rc;
 223     copy->op_status = event->op_status;
 224     pcmk__str_update((char **) &copy->output, event->output);
 225     copy->t_run = event->t_run;
 226     copy->t_rcchange = event->t_rcchange;
 227     copy->exec_time = event->exec_time;
 228     copy->queue_time = event->queue_time;
 229     copy->connection_rc = event->connection_rc;
 230     copy->params = pcmk__str_table_dup(event->params);
 231     pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
 232     pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);
 233 
 234     return copy;
 235 }
 236 
 237 /*!
 238  * \brief Free an executor event
 239  *
 240  * \param[in,out]  Executor event object to free
 241  */
 242 void
 243 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 244 {
 245     if (event == NULL) {
 246         return;
 247     }
 248     // @TODO Why are these const char *?
 249     free((void *) event->rsc_id);
 250     free((void *) event->op_type);
 251     free((void *) event->user_data);
 252     free((void *) event->remote_nodename);
 253     lrmd__reset_result(event);
 254     if (event->params != NULL) {
 255         g_hash_table_destroy(event->params);
 256     }
 257     free(event);
 258 }
 259 
 260 static void
 261 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 262 {
 263     const char *type;
 264     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 265     lrmd_private_t *native = lrmd->lrmd_private;
 266     lrmd_event_data_t event = { 0, };
 267 
 268     if (proxy_session != NULL) {
 269         /* this is proxy business */
 270         lrmd_internal_proxy_dispatch(lrmd, msg);
 271         return;
 272     } else if (!native->callback) {
 273         /* no callback set */
 274         crm_trace("notify event received but client has not set callback");
 275         return;
 276     }
 277 
 278     event.remote_nodename = native->remote_nodename;
 279     type = crm_element_value(msg, F_LRMD_OPERATION);
 280     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 281     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 282 
 283     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 284         event.type = lrmd_event_register;
 285     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 286         event.type = lrmd_event_unregister;
 287     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 288         time_t epoch = 0;
 289 
 290         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 291         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 292         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 293         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 294         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 295         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 296 
 297         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 298         event.t_run = (unsigned int) epoch;
 299 
 300         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 301         event.t_rcchange = (unsigned int) epoch;
 302 
 303         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 304         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 305 
 306         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 307         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 308         event.type = lrmd_event_exec_complete;
 309 
 310         /* output and exit_reason may be freed by a callback */
 311         event.output = crm_element_value_copy(msg, F_LRMD_RSC_OUTPUT);
 312         lrmd__set_result(&event, event.rc, event.op_status,
 313                          crm_element_value(msg, F_LRMD_RSC_EXIT_REASON));
 314 
 315         event.params = xml2list(msg);
 316     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 317         event.type = lrmd_event_new_client;
 318     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 319         event.type = lrmd_event_poke;
 320     } else {
 321         return;
 322     }
 323 
 324     crm_trace("op %s notify event received", type);
 325     native->callback(&event);
 326 
 327     if (event.params) {
 328         g_hash_table_destroy(event.params);
 329     }
 330     lrmd__reset_result(&event);
 331 }
 332 
 333 // \return Always 0, to indicate that IPC mainloop source should be kept
 334 static int
 335 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 336 {
 337     lrmd_t *lrmd = userdata;
 338     lrmd_private_t *native = lrmd->lrmd_private;
 339 
 340     if (native->callback != NULL) {
 341         xmlNode *msg = string2xml(buffer);
 342 
 343         lrmd_dispatch_internal(lrmd, msg);
 344         free_xml(msg);
 345     }
 346     return 0;
 347 }
 348 
 349 #ifdef HAVE_GNUTLS_GNUTLS_H
 350 static void
 351 lrmd_free_xml(gpointer userdata)
     /*  */
 352 {
 353     free_xml((xmlNode *) userdata);
 354 }
 355 
 356 static bool
 357 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 358 {
 359     lrmd_private_t *native = lrmd->lrmd_private;
 360 
 361     return (native->remote->tls_session != NULL);
 362 }
 363 
 364 /*!
 365  * \internal
 366  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 367  *
 368  * \param[in,out] userdata  API connection
 369  *
 370  * \return Always return a nonnegative value, which as a file descriptor
 371  *         dispatch function means keep the mainloop source, and as a
 372  *         trigger dispatch function, 0 means remove the trigger from the
 373  *         mainloop while 1 means keep it (and job completed)
 374  */
 375 static int
 376 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 377 {
 378     lrmd_t *lrmd = userdata;
 379     lrmd_private_t *native = lrmd->lrmd_private;
 380     xmlNode *xml = NULL;
 381     int rc = pcmk_rc_ok;
 382 
 383     if (!remote_executor_connected(lrmd)) {
 384         crm_trace("TLS dispatch triggered after disconnect");
 385         return 0;
 386     }
 387 
 388     crm_trace("TLS dispatch triggered");
 389 
 390     /* First check if there are any pending notifies to process that came
 391      * while we were waiting for replies earlier. */
 392     if (native->pending_notify) {
 393         GList *iter = NULL;
 394 
 395         crm_trace("Processing pending notifies");
 396         for (iter = native->pending_notify; iter; iter = iter->next) {
 397             lrmd_dispatch_internal(lrmd, iter->data);
 398         }
 399         g_list_free_full(native->pending_notify, lrmd_free_xml);
 400         native->pending_notify = NULL;
 401     }
 402 
 403     /* Next read the current buffer and see if there are any messages to handle. */
 404     switch (pcmk__remote_ready(native->remote, 0)) {
 405         case pcmk_rc_ok:
 406             rc = pcmk__read_remote_message(native->remote, -1);
 407             xml = pcmk__remote_message_xml(native->remote);
 408             break;
 409         case ETIME:
 410             // Nothing to read, check if a full message is already in buffer
 411             xml = pcmk__remote_message_xml(native->remote);
 412             break;
 413         default:
 414             rc = ENOTCONN;
 415             break;
 416     }
 417     while (xml) {
 418         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 419         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 420             lrmd_dispatch_internal(lrmd, xml);
 421         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 422             if (native->expected_late_replies > 0) {
 423                 native->expected_late_replies--;
 424             } else {
 425                 int reply_id = 0;
 426                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 427                 /* if this happens, we want to know about it */
 428                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 429             }
 430         }
 431         free_xml(xml);
 432         xml = pcmk__remote_message_xml(native->remote);
 433     }
 434 
 435     if (rc == ENOTCONN) {
 436         crm_info("Lost %s executor connection while reading data",
 437                  (native->remote_nodename? native->remote_nodename : "local"));
 438         lrmd_tls_disconnect(lrmd);
 439         return 0;
 440     }
 441     return 1;
 442 }
 443 #endif
 444 
 445 /* Not used with mainloop */
 446 int
 447 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 448 {
 449     lrmd_private_t *native = lrmd->lrmd_private;
 450 
 451     switch (native->type) {
 452         case pcmk__client_ipc:
 453             return crm_ipc_ready(native->ipc);
 454 
 455 #ifdef HAVE_GNUTLS_GNUTLS_H
 456         case pcmk__client_tls:
 457             if (native->pending_notify) {
 458                 return 1;
 459             } else {
 460                 int rc = pcmk__remote_ready(native->remote, 0);
 461 
 462                 switch (rc) {
 463                     case pcmk_rc_ok:
 464                         return 1;
 465                     case ETIME:
 466                         return 0;
 467                     default:
 468                         return pcmk_rc2legacy(rc);
 469                 }
 470             }
 471 #endif
 472         default:
 473             crm_err("Unsupported executor connection type (bug?): %d",
 474                     native->type);
 475             return -EPROTONOSUPPORT;
 476     }
 477 }
 478 
 479 /* Not used with mainloop */
 480 bool
 481 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 482 {
 483     lrmd_private_t *private = NULL;
 484 
 485     CRM_ASSERT(lrmd != NULL);
 486 
 487     private = lrmd->lrmd_private;
 488     switch (private->type) {
 489         case pcmk__client_ipc:
 490             while (crm_ipc_ready(private->ipc)) {
 491                 if (crm_ipc_read(private->ipc) > 0) {
 492                     const char *msg = crm_ipc_buffer(private->ipc);
 493 
 494                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 495                 }
 496             }
 497             break;
 498 #ifdef HAVE_GNUTLS_GNUTLS_H
 499         case pcmk__client_tls:
 500             lrmd_tls_dispatch(lrmd);
 501             break;
 502 #endif
 503         default:
 504             crm_err("Unsupported executor connection type (bug?): %d",
 505                     private->type);
 506     }
 507 
 508     if (lrmd_api_is_connected(lrmd) == FALSE) {
 509         crm_err("Connection closed");
 510         return FALSE;
 511     }
 512 
 513     return TRUE;
 514 }
 515 
 516 static xmlNode *
 517 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 518                enum lrmd_call_options options)
 519 {
 520     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 521 
 522     CRM_CHECK(op_msg != NULL, return NULL);
 523     CRM_CHECK(token != NULL, return NULL);
 524 
 525     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 526     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 527     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 528     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 529     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 530     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 531 
 532     if (data != NULL) {
 533         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 534     }
 535 
 536     crm_trace("Created executor %s command with call options %.8lx (%d)",
 537               op, (long)options, options);
 538     return op_msg;
 539 }
 540 
 541 static void
 542 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 543 {
 544     lrmd_t *lrmd = userdata;
 545     lrmd_private_t *native = lrmd->lrmd_private;
 546 
 547     switch (native->type) {
 548         case pcmk__client_ipc:
 549             crm_info("Disconnected from local executor");
 550             break;
 551 #ifdef HAVE_GNUTLS_GNUTLS_H
 552         case pcmk__client_tls:
 553             crm_info("Disconnected from remote executor on %s",
 554                      native->remote_nodename);
 555             break;
 556 #endif
 557         default:
 558             crm_err("Unsupported executor connection type %d (bug?)",
 559                     native->type);
 560     }
 561 
 562     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 563     native->ipc = NULL;
 564     native->source = NULL;
 565 
 566     if (native->callback) {
 567         lrmd_event_data_t event = { 0, };
 568         event.type = lrmd_event_disconnect;
 569         event.remote_nodename = native->remote_nodename;
 570         native->callback(&event);
 571     }
 572 }
 573 
 574 #ifdef HAVE_GNUTLS_GNUTLS_H
 575 static void
 576 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 577 {
 578     lrmd_t *lrmd = userdata;
 579     lrmd_private_t *native = lrmd->lrmd_private;
 580 
 581     crm_info("TLS connection destroyed");
 582 
 583     if (native->remote->tls_session) {
 584         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 585         gnutls_deinit(*native->remote->tls_session);
 586         gnutls_free(native->remote->tls_session);
 587     }
 588     if (native->psk_cred_c) {
 589         gnutls_psk_free_client_credentials(native->psk_cred_c);
 590     }
 591     if (native->sock) {
 592         close(native->sock);
 593     }
 594     if (native->process_notify) {
 595         mainloop_destroy_trigger(native->process_notify);
 596         native->process_notify = NULL;
 597     }
 598     if (native->pending_notify) {
 599         g_list_free_full(native->pending_notify, lrmd_free_xml);
 600         native->pending_notify = NULL;
 601     }
 602 
 603     free(native->remote->buffer);
 604     free(native->remote->start_state);
 605     native->remote->buffer = NULL;
 606     native->remote->start_state = NULL;
 607     native->source = 0;
 608     native->sock = 0;
 609     native->psk_cred_c = NULL;
 610     native->remote->tls_session = NULL;
 611     native->sock = 0;
 612 
 613     if (native->callback) {
 614         lrmd_event_data_t event = { 0, };
 615         event.remote_nodename = native->remote_nodename;
 616         event.type = lrmd_event_disconnect;
 617         native->callback(&event);
 618     }
 619     return;
 620 }
 621 
 622 // \return Standard Pacemaker return code
 623 int
 624 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 625                       const char *msg_type)
 626 {
 627     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 628     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 629     return pcmk__remote_send_xml(session, msg);
 630 }
 631 
 632 // \return Standard Pacemaker return code
 633 static int
 634 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 635                   xmlNode **reply)
 636 {
 637     lrmd_private_t *native = lrmd->lrmd_private;
 638     time_t start = time(NULL);
 639     const char *msg_type = NULL;
 640     int reply_id = 0;
 641     int remaining_timeout = 0;
 642     int rc = pcmk_rc_ok;
 643 
 644     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 645      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 646     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 647         total_timeout = MAX_TLS_RECV_WAIT;
 648     }
 649 
 650     for (*reply = NULL; *reply == NULL; ) {
 651 
 652         *reply = pcmk__remote_message_xml(native->remote);
 653         if (*reply == NULL) {
 654             /* read some more off the tls buffer if we still have time left. */
 655             if (remaining_timeout) {
 656                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 657             } else {
 658                 remaining_timeout = total_timeout;
 659             }
 660             if (remaining_timeout <= 0) {
 661                 return ETIME;
 662             }
 663 
 664             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 665             if (rc != pcmk_rc_ok) {
 666                 return rc;
 667             }
 668 
 669             *reply = pcmk__remote_message_xml(native->remote);
 670             if (*reply == NULL) {
 671                 return ENOMSG;
 672             }
 673         }
 674 
 675         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
 676         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 677 
 678         if (!msg_type) {
 679             crm_err("Empty msg type received while waiting for reply");
 680             free_xml(*reply);
 681             *reply = NULL;
 682         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 683             /* got a notify while waiting for reply, trigger the notify to be processed later */
 684             crm_info("queueing notify");
 685             native->pending_notify = g_list_append(native->pending_notify, *reply);
 686             if (native->process_notify) {
 687                 crm_info("notify trigger set.");
 688                 mainloop_set_trigger(native->process_notify);
 689             }
 690             *reply = NULL;
 691         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 692             /* msg isn't a reply, make some noise */
 693             crm_err("Expected a reply, got %s", msg_type);
 694             free_xml(*reply);
 695             *reply = NULL;
 696         } else if (reply_id != expected_reply_id) {
 697             if (native->expected_late_replies > 0) {
 698                 native->expected_late_replies--;
 699             } else {
 700                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 701             }
 702             free_xml(*reply);
 703             *reply = NULL;
 704         }
 705     }
 706 
 707     if (native->remote->buffer && native->process_notify) {
 708         mainloop_set_trigger(native->process_notify);
 709     }
 710 
 711     return rc;
 712 }
 713 
 714 // \return Standard Pacemaker return code
 715 static int
 716 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 717 {
 718     int rc = pcmk_rc_ok;
 719     lrmd_private_t *native = lrmd->lrmd_private;
 720 
 721     global_remote_msg_id++;
 722     if (global_remote_msg_id <= 0) {
 723         global_remote_msg_id = 1;
 724     }
 725 
 726     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 727                                "request");
 728     if (rc != pcmk_rc_ok) {
 729         crm_err("Disconnecting because TLS message could not be sent to "
 730                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 731         lrmd_tls_disconnect(lrmd);
 732     }
 733     return rc;
 734 }
 735 
 736 static int
 737 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 738 {
 739     int rc = 0;
 740     xmlNode *xml = NULL;
 741 
 742     if (!remote_executor_connected(lrmd)) {
 743         return -ENOTCONN;
 744     }
 745 
 746     rc = send_remote_message(lrmd, msg);
 747     if (rc != pcmk_rc_ok) {
 748         return pcmk_rc2legacy(rc);
 749     }
 750 
 751     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 752     if (rc != pcmk_rc_ok) {
 753         crm_err("Disconnecting remote after request %d reply not received: %s "
 754                 CRM_XS " rc=%d timeout=%dms",
 755                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 756         lrmd_tls_disconnect(lrmd);
 757     }
 758 
 759     if (reply) {
 760         *reply = xml;
 761     } else {
 762         free_xml(xml);
 763     }
 764 
 765     return pcmk_rc2legacy(rc);
 766 }
 767 #endif
 768 
 769 static int
 770 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 771 {
 772     int rc = pcmk_ok;
 773     lrmd_private_t *native = lrmd->lrmd_private;
 774 
 775     switch (native->type) {
 776         case pcmk__client_ipc:
 777             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 778             break;
 779 #ifdef HAVE_GNUTLS_GNUTLS_H
 780         case pcmk__client_tls:
 781             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 782             break;
 783 #endif
 784         default:
 785             crm_err("Unsupported executor connection type (bug?): %d",
 786                     native->type);
 787             rc = -EPROTONOSUPPORT;
 788     }
 789 
 790     return rc;
 791 }
 792 
 793 static int
 794 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 795 {
 796     int rc = pcmk_ok;
 797     lrmd_private_t *native = lrmd->lrmd_private;
 798 
 799     switch (native->type) {
 800         case pcmk__client_ipc:
 801             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 802             break;
 803 #ifdef HAVE_GNUTLS_GNUTLS_H
 804         case pcmk__client_tls:
 805             rc = send_remote_message(lrmd, msg);
 806             if (rc == pcmk_rc_ok) {
 807                 /* we don't want to wait around for the reply, but
 808                  * since the request/reply protocol needs to behave the same
 809                  * as libqb, a reply will eventually come later anyway. */
 810                 native->expected_late_replies++;
 811             }
 812             rc = pcmk_rc2legacy(rc);
 813             break;
 814 #endif
 815         default:
 816             crm_err("Unsupported executor connection type (bug?): %d",
 817                     native->type);
 818             rc = -EPROTONOSUPPORT;
 819     }
 820 
 821     return rc;
 822 }
 823 
 824 static int
 825 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 826 {
 827     lrmd_private_t *native = lrmd->lrmd_private;
 828 
 829     switch (native->type) {
 830         case pcmk__client_ipc:
 831             return crm_ipc_connected(native->ipc);
 832 #ifdef HAVE_GNUTLS_GNUTLS_H
 833         case pcmk__client_tls:
 834             return remote_executor_connected(lrmd);
 835 #endif
 836         default:
 837             crm_err("Unsupported executor connection type (bug?): %d",
 838                     native->type);
 839             return 0;
 840     }
 841 }
 842 
 843 /*!
 844  * \internal
 845  * \brief Send a prepared API command to the executor
 846  *
 847  * \param[in,out] lrmd          Existing connection to the executor
 848  * \param[in]     op            Name of API command to send
 849  * \param[in]     data          Command data XML to add to the sent command
 850  * \param[out]    output_data   If expecting a reply, it will be stored here
 851  * \param[in]     timeout       Timeout in milliseconds (if 0, defaults to
 852  *                              a sensible value per the type of connection,
 853  *                              standard vs. pacemaker remote);
 854  *                              also propagated to the command XML
 855  * \param[in]     call_options  Call options to pass to server when sending
 856  * \param[in]     expect_reply  If TRUE, wait for a reply from the server;
 857  *                              must be TRUE for IPC (as opposed to TLS) clients
 858  *
 859  * \return pcmk_ok on success, -errno on error
 860  */
 861 static int
 862 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 863                   xmlNode **output_data, int timeout,
 864                   enum lrmd_call_options options, gboolean expect_reply)
 865 {
 866     int rc = pcmk_ok;
 867     lrmd_private_t *native = lrmd->lrmd_private;
 868     xmlNode *op_msg = NULL;
 869     xmlNode *op_reply = NULL;
 870 
 871     if (!lrmd_api_is_connected(lrmd)) {
 872         return -ENOTCONN;
 873     }
 874 
 875     if (op == NULL) {
 876         crm_err("No operation specified");
 877         return -EINVAL;
 878     }
 879 
 880     CRM_CHECK(native->token != NULL,;
 881         );
 882     crm_trace("Sending %s op to executor", op);
 883 
 884     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 885 
 886     if (op_msg == NULL) {
 887         return -EINVAL;
 888     }
 889 
 890     if (expect_reply) {
 891         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 892     } else {
 893         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 894         goto done;
 895     }
 896 
 897     if (rc < 0) {
 898         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 899         goto done;
 900 
 901     } else if(op_reply == NULL) {
 902         rc = -ENOMSG;
 903         goto done;
 904     }
 905 
 906     rc = pcmk_ok;
 907     crm_trace("%s op reply received", op);
 908     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 909         rc = -ENOMSG;
 910         goto done;
 911     }
 912 
 913     crm_log_xml_trace(op_reply, "Reply");
 914 
 915     if (output_data) {
 916         *output_data = op_reply;
 917         op_reply = NULL;        /* Prevent subsequent free */
 918     }
 919 
 920   done:
 921     if (lrmd_api_is_connected(lrmd) == FALSE) {
 922         crm_err("Executor disconnected");
 923     }
 924 
 925     free_xml(op_msg);
 926     free_xml(op_reply);
 927     return rc;
 928 }
 929 
 930 static int
 931 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 932 {
 933     int rc;
 934     lrmd_private_t *native = lrmd->lrmd_private;
 935     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 936 
 937     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 938     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 939                            (native->type == pcmk__client_ipc));
 940     free_xml(data);
 941 
 942     return rc < 0 ? rc : pcmk_ok;
 943 }
 944 
 945 // \return Standard Pacemaker return code
 946 int
 947 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 948 {
 949     int rc = pcmk_rc_ok;
 950     const char *value;
 951     lrmd_private_t *native = lrmd->lrmd_private;
 952     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 953 
 954     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 955 
 956     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 957     if ((value) &&
 958         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 959        crm_xml_add(data, F_LRMD_WATCHDOG, value);
 960     }
 961 
 962     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 963                            (native->type == pcmk__client_ipc));
 964     free_xml(data);
 965     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 966 }
 967 
 968 static int
 969 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 970 {
 971     int rc = pcmk_ok;
 972     lrmd_private_t *native = lrmd->lrmd_private;
 973     xmlNode *reply = NULL;
 974     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 975 
 976     crm_xml_add(hello, F_TYPE, T_LRMD);
 977     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 978     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 979     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 980 
 981     /* advertise that we are a proxy provider */
 982     if (native->proxy_callback) {
 983         pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
 984     }
 985 
 986     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 987 
 988     if (rc < 0) {
 989         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 990         rc = -ECOMM;
 991     } else if (reply == NULL) {
 992         crm_err("Did not receive registration reply");
 993         rc = -EPROTO;
 994     } else {
 995         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 996         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 997         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 998         const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
 999         long long uptime = -1;
1000 
1001         crm_element_value_int(reply, F_LRMD_RC, &rc);
1002 
1003         /* The remote executor may add its uptime to the XML reply, which is
1004          * useful in handling transient attributes when the connection to the
1005          * remote node unexpectedly drops.  If no parameter is given, just
1006          * default to -1.
1007          */
1008         crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
1009         native->remote->uptime = uptime;
1010 
1011         if (start_state) {
1012             native->remote->start_state = strdup(start_state);
1013         }
1014 
1015         if (rc == -EPROTO) {
1016             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
1017                 LRMD_PROTOCOL_VERSION, version);
1018             crm_log_xml_err(reply, "Protocol Error");
1019 
1020         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
1021             crm_err("Invalid registration message: %s", msg_type);
1022             crm_log_xml_err(reply, "Bad reply");
1023             rc = -EPROTO;
1024         } else if (tmp_ticket == NULL) {
1025             crm_err("No registration token provided");
1026             crm_log_xml_err(reply, "Bad reply");
1027             rc = -EPROTO;
1028         } else {
1029             crm_trace("Obtained registration token: %s", tmp_ticket);
1030             native->token = strdup(tmp_ticket);
1031             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1032             rc = pcmk_ok;
1033         }
1034     }
1035 
1036     free_xml(reply);
1037     free_xml(hello);
1038 
1039     if (rc != pcmk_ok) {
1040         lrmd_api_disconnect(lrmd);
1041     }
1042     return rc;
1043 }
1044 
1045 static int
1046 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1047 {
1048     int rc = pcmk_ok;
1049     lrmd_private_t *native = lrmd->lrmd_private;
1050 
1051     struct ipc_client_callbacks lrmd_callbacks = {
1052         .dispatch = lrmd_ipc_dispatch,
1053         .destroy = lrmd_ipc_connection_destroy
1054     };
1055 
1056     crm_info("Connecting to executor");
1057 
1058     if (fd) {
1059         /* No mainloop */
1060         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1061         if (native->ipc != NULL) {
1062             rc = pcmk__connect_generic_ipc(native->ipc);
1063             if (rc == pcmk_rc_ok) {
1064                 rc = pcmk__ipc_fd(native->ipc, fd);
1065             }
1066             if (rc != pcmk_rc_ok) {
1067                 crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
1068                 rc = -ENOTCONN;
1069             }
1070         }
1071     } else {
1072         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1073         native->ipc = mainloop_get_ipc_client(native->source);
1074     }
1075 
1076     if (native->ipc == NULL) {
1077         crm_debug("Could not connect to the executor API");
1078         rc = -ENOTCONN;
1079     }
1080 
1081     return rc;
1082 }
1083 
1084 #ifdef HAVE_GNUTLS_GNUTLS_H
1085 static void
1086 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1087 {
1088     CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
1089 
1090     dest->data = gnutls_malloc(source->size);
1091     CRM_ASSERT(dest->data);
1092 
1093     memcpy(dest->data, source->data, source->size);
1094     dest->size = source->size;
1095 }
1096 
1097 static void
1098 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1099 {
1100     gnutls_free(datum->data);
1101     datum->data = NULL;
1102     datum->size = 0;
1103 }
1104 
1105 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1106 
1107 // \return Standard Pacemaker return code
1108 static int
1109 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1110 {
1111     FILE *stream = NULL;
1112     size_t buf_len = KEY_READ_LEN;
1113 
1114     if ((location == NULL) || (key == NULL)) {
1115         return EINVAL;
1116     }
1117 
1118     stream = fopen(location, "r");
1119     if (stream == NULL) {
1120         return errno;
1121     }
1122 
1123     key->data = gnutls_malloc(buf_len);
1124     key->size = 0;
1125     while (!feof(stream)) {
1126         int next = fgetc(stream);
1127 
1128         if (next == EOF) {
1129             if (!feof(stream)) {
1130                 crm_warn("Pacemaker Remote key read was partially successful "
1131                          "(copy in memory may be corrupted)");
1132             }
1133             break;
1134         }
1135         if (key->size == buf_len) {
1136             buf_len = key->size + KEY_READ_LEN;
1137             key->data = gnutls_realloc(key->data, buf_len);
1138             CRM_ASSERT(key->data);
1139         }
1140         key->data[key->size++] = (unsigned char) next;
1141     }
1142     fclose(stream);
1143 
1144     if (key->size == 0) {
1145         clear_gnutls_datum(key);
1146         return ENOKEY;
1147     }
1148     return pcmk_rc_ok;
1149 }
1150 
1151 // Cache the most recently used Pacemaker Remote authentication key
1152 
1153 struct key_cache_s {
1154     time_t updated;         // When cached key was read (valid for 1 minute)
1155     const char *location;   // Where cached key was read from
1156     gnutls_datum_t key;     // Cached key
1157 };
1158 
1159 static bool
1160 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1161 {
1162     return key_cache->updated != 0;
1163 }
1164 
1165 static bool
1166 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1167 {
1168     return (time(NULL) - key_cache->updated) >= 60;
1169 }
1170 
1171 static void
1172 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1173 {
1174     clear_gnutls_datum(&(key_cache->key));
1175     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1176         key_cache->updated = 0;
1177         key_cache->location = NULL;
1178         crm_debug("Cleared Pacemaker Remote key cache");
1179     }
1180 }
1181 
1182 static void
1183 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1184 {
1185     copy_gnutls_datum(key, &(key_cache->key));
1186     crm_debug("Using cached Pacemaker Remote key from %s",
1187               pcmk__s(key_cache->location, "unknown location"));
1188 }
1189 
1190 static void
1191 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1192           const char *location)
1193 {
1194     key_cache->updated = time(NULL);
1195     key_cache->location = location;
1196     copy_gnutls_datum(&(key_cache->key), key);
1197     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1198               pcmk__s(location, "unknown location"));
1199 }
1200 
1201 /*!
1202  * \internal
1203  * \brief Get Pacemaker Remote authentication key from file or cache
1204  *
1205  * \param[in]  location         Path to key file to try (this memory must
1206  *                              persist across all calls of this function)
1207  * \param[out] key              Key from location or cache
1208  *
1209  * \return Standard Pacemaker return code
1210  */
1211 static int
1212 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1213 {
1214     static struct key_cache_s key_cache = { 0, };
1215     int rc = pcmk_rc_ok;
1216 
1217     if ((location == NULL) || (key == NULL)) {
1218         return EINVAL;
1219     }
1220 
1221     if (key_is_cached(&key_cache)) {
1222         if (key_cache_expired(&key_cache)) {
1223             clear_key_cache(&key_cache);
1224         } else {
1225             get_cached_key(&key_cache, key);
1226             return pcmk_rc_ok;
1227         }
1228     }
1229 
1230     rc = read_gnutls_key(location, key);
1231     if (rc != pcmk_rc_ok) {
1232         return rc;
1233     }
1234     cache_key(&key_cache, key, location);
1235     return pcmk_rc_ok;
1236 }
1237 
1238 /*!
1239  * \internal
1240  * \brief Initialize the Pacemaker Remote authentication key
1241  *
1242  * Try loading the Pacemaker Remote authentication key from cache if available,
1243  * otherwise from these locations, in order of preference: the value of the
1244  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1245  * file location; or (for historical reasons) /etc/corosync/authkey.
1246  *
1247  * \param[out] key  Where to store key
1248  *
1249  * \return Standard Pacemaker return code
1250  */
1251 int
1252 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1253 {
1254     static const char *env_location = NULL;
1255     static bool need_env = true;
1256 
1257     int env_rc = pcmk_rc_ok;
1258     int default_rc = pcmk_rc_ok;
1259     int alt_rc = pcmk_rc_ok;
1260 
1261     bool env_is_default = false;
1262     bool env_is_fallback = false;
1263 
1264     if (need_env) {
1265         env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
1266         need_env = false;
1267     }
1268 
1269     // Try location in environment variable, if set
1270     if (env_location != NULL) {
1271         env_rc = get_remote_key(env_location, key);
1272         if (env_rc == pcmk_rc_ok) {
1273             return pcmk_rc_ok;
1274         }
1275 
1276         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1277         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1278 
1279         /* @TODO It would be more secure to fail, rather than fall back to the
1280          * default, if an explicitly set key location is not readable, and it
1281          * would be better to never use the Corosync location as a fallback.
1282          * However, that would break any deployments currently working with the
1283          * fallbacks.
1284          */
1285     }
1286 
1287     // Try default location, if environment wasn't explicitly set to it
1288     if (env_is_default) {
1289         default_rc = env_rc;
1290     } else {
1291         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1292     }
1293 
1294     // Try fallback location, if environment wasn't set to it and default failed
1295     if (env_is_fallback) {
1296         alt_rc = env_rc;
1297     } else if (default_rc != pcmk_rc_ok) {
1298         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1299     }
1300 
1301     // We have all results, so log and return
1302 
1303     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1304         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1305 
1306         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1307                  env_location,
1308                  env_is_default? "" : "or default location ",
1309                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1310                  !env_is_default && !env_is_fallback? " " : "",
1311                  env_is_fallback? "" : "or fallback location ",
1312                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1313                  pcmk_rc_str(env_rc));
1314         return ENOKEY;
1315     }
1316 
1317     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1318         crm_warn("Could not read Pacemaker Remote key from %s "
1319                  "(using %s location %s instead): %s",
1320                  env_location,
1321                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1322                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1323                  pcmk_rc_str(env_rc));
1324         return pcmk_rc_ok;
1325     }
1326 
1327     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
1328         // Environment unset, defaults failed
1329         crm_warn("Could not read Pacemaker Remote key from default location %s"
1330                  " (or fallback location %s): %s",
1331                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1332                  pcmk_rc_str(default_rc));
1333         return ENOKEY;
1334     }
1335 
1336     return pcmk_rc_ok; // Environment variable unset, a default worked
1337 }
1338 
1339 static void
1340 lrmd_gnutls_global_init(void)
     /*  */
1341 {
1342     static int gnutls_init = 0;
1343 
1344     if (!gnutls_init) {
1345         crm_gnutls_global_init();
1346     }
1347     gnutls_init = 1;
1348 }
1349 #endif
1350 
1351 static void
1352 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1353 {
1354     lrmd_private_t *native = lrmd->lrmd_private;
1355 
1356     if (native->callback) {
1357         lrmd_event_data_t event = { 0, };
1358         event.type = lrmd_event_connect;
1359         event.remote_nodename = native->remote_nodename;
1360         event.connection_rc = rc;
1361         native->callback(&event);
1362     }
1363 }
1364 
1365 #ifdef HAVE_GNUTLS_GNUTLS_H
1366 static inline int
1367 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1368 {
1369     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1370 }
1371 
1372 /*!
1373  * \internal
1374  * \brief Add trigger and file descriptor mainloop sources for TLS
1375  *
1376  * \param[in,out] lrmd          API connection with established TLS session
1377  * \param[in]     do_handshake  Whether to perform executor handshake
1378  *
1379  * \return Standard Pacemaker return code
1380  */
1381 static int
1382 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1383 {
1384     lrmd_private_t *native = lrmd->lrmd_private;
1385     int rc = pcmk_rc_ok;
1386 
1387     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1388                                    native->server, native->port);
1389 
1390     struct mainloop_fd_callbacks tls_fd_callbacks = {
1391         .dispatch = lrmd_tls_dispatch,
1392         .destroy = lrmd_tls_connection_destroy,
1393     };
1394 
1395     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1396                                                   lrmd_tls_dispatch, lrmd);
1397     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1398                                      &tls_fd_callbacks);
1399 
1400     /* Async connections lose the client name provided by the API caller, so we
1401      * have to use our generated name here to perform the executor handshake.
1402      *
1403      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1404      * that name in this function instead of generating one anyway.
1405      */
1406     if (do_handshake) {
1407         rc = lrmd_handshake(lrmd, name);
1408         rc = pcmk_legacy2rc(rc);
1409     }
1410     free(name);
1411     return rc;
1412 }
1413 
1414 static void
1415 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1416 {
1417     lrmd_t *lrmd = userdata;
1418     lrmd_private_t *native = lrmd->lrmd_private;
1419     gnutls_datum_t psk_key = { NULL, 0 };
1420 
1421     native->async_timer = 0;
1422 
1423     if (rc != pcmk_rc_ok) {
1424         lrmd_tls_connection_destroy(lrmd);
1425         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1426                  CRM_XS " rc=%d",
1427                  native->server, native->port, pcmk_rc_str(rc), rc);
1428         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1429         return;
1430     }
1431 
1432     /* The TCP connection was successful, so establish the TLS connection.
1433      * @TODO make this async to avoid blocking code in client
1434      */
1435 
1436     native->sock = sock;
1437 
1438     rc = lrmd__init_remote_key(&psk_key);
1439     if (rc != pcmk_rc_ok) {
1440         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1441                  CRM_XS " rc=%d",
1442                  native->server, native->port, pcmk_rc_str(rc), rc);
1443         lrmd_tls_connection_destroy(lrmd);
1444         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1445         return;
1446     }
1447 
1448     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1449     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1450     gnutls_free(psk_key.data);
1451 
1452     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1453                                                         GNUTLS_CRD_PSK,
1454                                                         native->psk_cred_c);
1455     if (native->remote->tls_session == NULL) {
1456         lrmd_tls_connection_destroy(lrmd);
1457         report_async_connection_result(lrmd, -EPROTO);
1458         return;
1459     }
1460 
1461     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1462         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1463                  native->server, native->port);
1464         gnutls_deinit(*native->remote->tls_session);
1465         gnutls_free(native->remote->tls_session);
1466         native->remote->tls_session = NULL;
1467         lrmd_tls_connection_destroy(lrmd);
1468         report_async_connection_result(lrmd, -EKEYREJECTED);
1469         return;
1470     }
1471 
1472     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1473              native->server, native->port);
1474     rc = add_tls_to_mainloop(lrmd, true);
1475     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1476 }
1477 
1478 static int
1479 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1480 {
1481     int rc;
1482     int timer_id = 0;
1483     lrmd_private_t *native = lrmd->lrmd_private;
1484 
1485     lrmd_gnutls_global_init();
1486     native->sock = -1;
1487     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1488                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1489     if (rc != pcmk_rc_ok) {
1490         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1491                  CRM_XS " rc=%d",
1492                  native->server, native->port, pcmk_rc_str(rc), rc);
1493         return pcmk_rc2legacy(rc);
1494     }
1495     native->async_timer = timer_id;
1496     return pcmk_ok;
1497 }
1498 
1499 static int
1500 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1501 {
1502     int rc;
1503 
1504     lrmd_private_t *native = lrmd->lrmd_private;
1505     gnutls_datum_t psk_key = { NULL, 0 };
1506 
1507     lrmd_gnutls_global_init();
1508 
1509     native->sock = -1;
1510     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1511                               &(native->sock), NULL, NULL);
1512     if (rc != pcmk_rc_ok) {
1513         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1514                  CRM_XS " rc=%d",
1515                  native->server, native->port, pcmk_rc_str(rc), rc);
1516         lrmd_tls_connection_destroy(lrmd);
1517         return -ENOTCONN;
1518     }
1519 
1520     rc = lrmd__init_remote_key(&psk_key);
1521     if (rc != pcmk_rc_ok) {
1522         lrmd_tls_connection_destroy(lrmd);
1523         return pcmk_rc2legacy(rc);
1524     }
1525 
1526     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1527     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1528     gnutls_free(psk_key.data);
1529 
1530     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1531                                                         GNUTLS_CRD_PSK,
1532                                                         native->psk_cred_c);
1533     if (native->remote->tls_session == NULL) {
1534         lrmd_tls_connection_destroy(lrmd);
1535         return -EPROTO;
1536     }
1537 
1538     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1539         crm_err("Session creation for %s:%d failed", native->server, native->port);
1540         gnutls_deinit(*native->remote->tls_session);
1541         gnutls_free(native->remote->tls_session);
1542         native->remote->tls_session = NULL;
1543         lrmd_tls_connection_destroy(lrmd);
1544         return -EKEYREJECTED;
1545     }
1546 
1547     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1548              native->port);
1549 
1550     if (fd) {
1551         *fd = native->sock;
1552     } else {
1553         add_tls_to_mainloop(lrmd, false);
1554     }
1555     return pcmk_ok;
1556 }
1557 #endif
1558 
1559 static int
1560 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1561 {
1562     int rc = -ENOTCONN;
1563     lrmd_private_t *native = lrmd->lrmd_private;
1564 
1565     switch (native->type) {
1566         case pcmk__client_ipc:
1567             rc = lrmd_ipc_connect(lrmd, fd);
1568             break;
1569 #ifdef HAVE_GNUTLS_GNUTLS_H
1570         case pcmk__client_tls:
1571             rc = lrmd_tls_connect(lrmd, fd);
1572             break;
1573 #endif
1574         default:
1575             crm_err("Unsupported executor connection type (bug?): %d",
1576                     native->type);
1577             rc = -EPROTONOSUPPORT;
1578     }
1579 
1580     if (rc == pcmk_ok) {
1581         rc = lrmd_handshake(lrmd, name);
1582     }
1583 
1584     return rc;
1585 }
1586 
1587 static int
1588 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1589 {
1590     int rc = pcmk_ok;
1591     lrmd_private_t *native = lrmd->lrmd_private;
1592 
1593     CRM_CHECK(native && native->callback, return -EINVAL);
1594 
1595     switch (native->type) {
1596         case pcmk__client_ipc:
1597             /* fake async connection with ipc.  it should be fast
1598              * enough that we gain very little from async */
1599             rc = lrmd_api_connect(lrmd, name, NULL);
1600             if (!rc) {
1601                 report_async_connection_result(lrmd, rc);
1602             }
1603             break;
1604 #ifdef HAVE_GNUTLS_GNUTLS_H
1605         case pcmk__client_tls:
1606             rc = lrmd_tls_connect_async(lrmd, timeout);
1607             if (rc) {
1608                 /* connection failed, report rc now */
1609                 report_async_connection_result(lrmd, rc);
1610             }
1611             break;
1612 #endif
1613         default:
1614             crm_err("Unsupported executor connection type (bug?): %d",
1615                     native->type);
1616             rc = -EPROTONOSUPPORT;
1617     }
1618 
1619     return rc;
1620 }
1621 
1622 static void
1623 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1624 {
1625     lrmd_private_t *native = lrmd->lrmd_private;
1626 
1627     if (native->source != NULL) {
1628         /* Attached to mainloop */
1629         mainloop_del_ipc_client(native->source);
1630         native->source = NULL;
1631         native->ipc = NULL;
1632 
1633     } else if (native->ipc) {
1634         /* Not attached to mainloop */
1635         crm_ipc_t *ipc = native->ipc;
1636 
1637         native->ipc = NULL;
1638         crm_ipc_close(ipc);
1639         crm_ipc_destroy(ipc);
1640     }
1641 }
1642 
1643 #ifdef HAVE_GNUTLS_GNUTLS_H
1644 static void
1645 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1646 {
1647     lrmd_private_t *native = lrmd->lrmd_private;
1648 
1649     if (native->remote->tls_session) {
1650         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1651         gnutls_deinit(*native->remote->tls_session);
1652         gnutls_free(native->remote->tls_session);
1653         native->remote->tls_session = 0;
1654     }
1655 
1656     if (native->async_timer) {
1657         g_source_remove(native->async_timer);
1658         native->async_timer = 0;
1659     }
1660 
1661     if (native->source != NULL) {
1662         /* Attached to mainloop */
1663         mainloop_del_ipc_client(native->source);
1664         native->source = NULL;
1665 
1666     } else if (native->sock) {
1667         close(native->sock);
1668         native->sock = 0;
1669     }
1670 
1671     if (native->pending_notify) {
1672         g_list_free_full(native->pending_notify, lrmd_free_xml);
1673         native->pending_notify = NULL;
1674     }
1675 }
1676 #endif
1677 
1678 static int
1679 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1680 {
1681     lrmd_private_t *native = lrmd->lrmd_private;
1682     int rc = pcmk_ok;
1683 
1684     switch (native->type) {
1685         case pcmk__client_ipc:
1686             crm_debug("Disconnecting from local executor");
1687             lrmd_ipc_disconnect(lrmd);
1688             break;
1689 #ifdef HAVE_GNUTLS_GNUTLS_H
1690         case pcmk__client_tls:
1691             crm_debug("Disconnecting from remote executor on %s",
1692                       native->remote_nodename);
1693             lrmd_tls_disconnect(lrmd);
1694             break;
1695 #endif
1696         default:
1697             crm_err("Unsupported executor connection type (bug?): %d",
1698                     native->type);
1699             rc = -EPROTONOSUPPORT;
1700     }
1701 
1702     free(native->token);
1703     native->token = NULL;
1704 
1705     free(native->peer_version);
1706     native->peer_version = NULL;
1707     return rc;
1708 }
1709 
1710 static int
1711 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1712                       const char *rsc_id,
1713                       const char *class,
1714                       const char *provider, const char *type, enum lrmd_call_options options)
1715 {
1716     int rc = pcmk_ok;
1717     xmlNode *data = NULL;
1718 
1719     if (!class || !type || !rsc_id) {
1720         return -EINVAL;
1721     }
1722     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1723         && (provider == NULL)) {
1724         return -EINVAL;
1725     }
1726 
1727     data = create_xml_node(NULL, F_LRMD_RSC);
1728 
1729     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1730     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1731     crm_xml_add(data, F_LRMD_CLASS, class);
1732     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1733     crm_xml_add(data, F_LRMD_TYPE, type);
1734     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1735     free_xml(data);
1736 
1737     return rc;
1738 }
1739 
1740 static int
1741 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1742 {
1743     int rc = pcmk_ok;
1744     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1745 
1746     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1747     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1748     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1749     free_xml(data);
1750 
1751     return rc;
1752 }
1753 
1754 lrmd_rsc_info_t *
1755 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1756                   const char *provider, const char *type)
1757 {
1758     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1759 
1760     CRM_ASSERT(rsc_info);
1761     pcmk__str_update(&rsc_info->id, rsc_id);
1762     pcmk__str_update(&rsc_info->standard, standard);
1763     pcmk__str_update(&rsc_info->provider, provider);
1764     pcmk__str_update(&rsc_info->type, type);
1765     return rsc_info;
1766 }
1767 
1768 lrmd_rsc_info_t *
1769 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1770 {
1771     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1772                              rsc_info->provider, rsc_info->type);
1773 }
1774 
1775 void
1776 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1777 {
1778     if (!rsc_info) {
1779         return;
1780     }
1781     free(rsc_info->id);
1782     free(rsc_info->type);
1783     free(rsc_info->standard);
1784     free(rsc_info->provider);
1785     free(rsc_info);
1786 }
1787 
1788 static lrmd_rsc_info_t *
1789 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1790 {
1791     lrmd_rsc_info_t *rsc_info = NULL;
1792     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1793     xmlNode *output = NULL;
1794     const char *class = NULL;
1795     const char *provider = NULL;
1796     const char *type = NULL;
1797 
1798     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1799     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1800     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1801     free_xml(data);
1802 
1803     if (!output) {
1804         return NULL;
1805     }
1806 
1807     class = crm_element_value(output, F_LRMD_CLASS);
1808     provider = crm_element_value(output, F_LRMD_PROVIDER);
1809     type = crm_element_value(output, F_LRMD_TYPE);
1810 
1811     if (!class || !type) {
1812         free_xml(output);
1813         return NULL;
1814     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1815                && !provider) {
1816         free_xml(output);
1817         return NULL;
1818     }
1819 
1820     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1821     free_xml(output);
1822     return rsc_info;
1823 }
1824 
1825 void
1826 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1827 {
1828     if (op_info) {
1829         free(op_info->rsc_id);
1830         free(op_info->action);
1831         free(op_info->interval_ms_s);
1832         free(op_info->timeout_ms_s);
1833         free(op_info);
1834     }
1835 }
1836 
1837 static int
1838 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1839                            enum lrmd_call_options options, GList **output)
1840 {
1841     xmlNode *data = NULL;
1842     xmlNode *output_xml = NULL;
1843     int rc = pcmk_ok;
1844 
1845     if (output == NULL) {
1846         return -EINVAL;
1847     }
1848     *output = NULL;
1849 
1850     // Send request
1851     if (rsc_id) {
1852         data = create_xml_node(NULL, F_LRMD_RSC);
1853         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1854         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1855     }
1856     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1857                            timeout_ms, options, TRUE);
1858     if (data) {
1859         free_xml(data);
1860     }
1861 
1862     // Process reply
1863     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1864         return rc;
1865     }
1866     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1867          (rsc_xml != NULL) && (rc == pcmk_ok);
1868          rsc_xml = crm_next_same_xml(rsc_xml)) {
1869 
1870         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1871         if (rsc_id == NULL) {
1872             crm_err("Could not parse recurring operation information from executor");
1873             continue;
1874         }
1875         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1876              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1877 
1878             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1879 
1880             if (op_info == NULL) {
1881                 rc = -ENOMEM;
1882                 break;
1883             }
1884             op_info->rsc_id = strdup(rsc_id);
1885             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1886             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1887                                                             F_LRMD_RSC_INTERVAL);
1888             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1889                                                            F_LRMD_TIMEOUT);
1890             *output = g_list_prepend(*output, op_info);
1891         }
1892     }
1893     free_xml(output_xml);
1894     return rc;
1895 }
1896 
1897 
1898 static void
1899 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1900 {
1901     lrmd_private_t *native = lrmd->lrmd_private;
1902 
1903     native->callback = callback;
1904 }
1905 
1906 void
1907 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1908 {
1909     lrmd_private_t *native = lrmd->lrmd_private;
1910 
1911     native->proxy_callback = callback;
1912     native->proxy_callback_userdata = userdata;
1913 }
1914 
1915 void
1916 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1917 {
1918     lrmd_private_t *native = lrmd->lrmd_private;
1919 
1920     if (native->proxy_callback) {
1921         crm_log_xml_trace(msg, "PROXY_INBOUND");
1922         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1923     }
1924 }
1925 
1926 int
1927 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1928 {
1929     if (lrmd == NULL) {
1930         return -ENOTCONN;
1931     }
1932     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1933 
1934     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1935     return lrmd_send_xml_no_reply(lrmd, msg);
1936 }
1937 
1938 static int
1939 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1940 {
1941     int rc = pcmk_ok;
1942     stonith_t *stonith_api = stonith_api_new();
1943 
1944     if (stonith_api == NULL) {
1945         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1946         return -ENOMEM;
1947     }
1948 
1949     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1950                                      provider, output, 0);
1951     if ((rc == pcmk_ok) && (*output == NULL)) {
1952         rc = -EIO;
1953     }
1954     stonith_api->cmds->free(stonith_api);
1955     return rc;
1956 }
1957 
1958 static int
1959 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1960                       const char *type, char **output,
1961                       enum lrmd_call_options options)
1962 {
1963     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1964                                            output, options, NULL);
1965 }
1966 
1967 static int
1968 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1969                              const char *provider, const char *type,
1970                              char **output, enum lrmd_call_options options,
1971                              lrmd_key_value_t *params)
1972 {
1973     svc_action_t *action = NULL;
1974     GHashTable *params_table = NULL;
1975 
1976     if (!standard || !type) {
1977         lrmd_key_value_freeall(params);
1978         return -EINVAL;
1979     }
1980 
1981     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1982         lrmd_key_value_freeall(params);
1983         return stonith_get_metadata(provider, type, output);
1984     }
1985 
1986     params_table = pcmk__strkey_table(free, free);
1987     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1988         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1989     }
1990     action = services__create_resource_action(type, standard, provider, type,
1991                                               PCMK_ACTION_META_DATA, 0,
1992                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
1993                                               params_table, 0);
1994     lrmd_key_value_freeall(params);
1995 
1996     if (action == NULL) {
1997         return -ENOMEM;
1998     }
1999     if (action->rc != PCMK_OCF_UNKNOWN) {
2000         services_action_free(action);
2001         return -EINVAL;
2002     }
2003 
2004     if (!services_action_sync(action)) {
2005         crm_err("Failed to retrieve meta-data for %s:%s:%s",
2006                 standard, provider, type);
2007         services_action_free(action);
2008         return -EIO;
2009     }
2010 
2011     if (!action->stdout_data) {
2012         crm_err("Failed to receive meta-data for %s:%s:%s",
2013                 standard, provider, type);
2014         services_action_free(action);
2015         return -EIO;
2016     }
2017 
2018     *output = strdup(action->stdout_data);
2019     services_action_free(action);
2020 
2021     return pcmk_ok;
2022 }
2023 
2024 static int
2025 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2026               const char *userdata, guint interval_ms,
2027               int timeout,      /* ms */
2028               int start_delay,  /* ms */
2029               enum lrmd_call_options options, lrmd_key_value_t * params)
2030 {
2031     int rc = pcmk_ok;
2032     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2033     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2034     lrmd_key_value_t *tmp = NULL;
2035 
2036     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2037     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2038     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2039     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
2040     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2041     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2042     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
2043 
2044     for (tmp = params; tmp; tmp = tmp->next) {
2045         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2046     }
2047 
2048     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2049     free_xml(data);
2050 
2051     lrmd_key_value_freeall(params);
2052     return rc;
2053 }
2054 
2055 /* timeout is in ms */
2056 static int
2057 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2058                     int timeout, lrmd_key_value_t *params)
2059 {
2060     int rc = pcmk_ok;
2061     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
2062     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2063     lrmd_key_value_t *tmp = NULL;
2064 
2065     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2066     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
2067     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
2068     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2069 
2070     for (tmp = params; tmp; tmp = tmp->next) {
2071         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2072     }
2073 
2074     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2075                            lrmd_opt_notify_orig_only, TRUE);
2076     free_xml(data);
2077 
2078     lrmd_key_value_freeall(params);
2079     return rc;
2080 }
2081 
2082 static int
2083 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2084                 guint interval_ms)
2085 {
2086     int rc = pcmk_ok;
2087     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2088 
2089     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2090     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2091     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2092     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2093     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2094     free_xml(data);
2095     return rc;
2096 }
2097 
2098 static int
2099 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2100 {
2101     int rc = 0;
2102     stonith_t *stonith_api = stonith_api_new();
2103     stonith_key_value_t *stonith_resources = NULL;
2104     stonith_key_value_t *dIter = NULL;
2105 
2106     if (stonith_api == NULL) {
2107         crm_err("Could not list fence agents: API memory allocation failed");
2108         return -ENOMEM;
2109     }
2110     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2111                                    &stonith_resources, 0);
2112     stonith_api->cmds->free(stonith_api);
2113 
2114     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2115         rc++;
2116         if (resources) {
2117             *resources = lrmd_list_add(*resources, dIter->value);
2118         }
2119     }
2120 
2121     stonith_key_value_freeall(stonith_resources, 1, 0);
2122     return rc;
2123 }
2124 
2125 static int
2126 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2127                      const char *provider)
2128 {
2129     int rc = 0;
2130     int stonith_count = 0; // Initially, whether to include stonith devices
2131 
2132     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2133         stonith_count = 1;
2134 
2135     } else {
2136         GList *gIter = NULL;
2137         GList *agents = resources_list_agents(class, provider);
2138 
2139         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2140             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2141             rc++;
2142         }
2143         g_list_free_full(agents, free);
2144 
2145         if (!class) {
2146             stonith_count = 1;
2147         }
2148     }
2149 
2150     if (stonith_count) {
2151         // Now, if stonith devices are included, how many there are
2152         stonith_count = list_stonith_agents(resources);
2153         if (stonith_count > 0) {
2154             rc += stonith_count;
2155         }
2156     }
2157     if (rc == 0) {
2158         crm_notice("No agents found for class %s", class);
2159         rc = -EPROTONOSUPPORT;
2160     }
2161     return rc;
2162 }
2163 
2164 static bool
2165 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2166 {
2167     bool found = false;
2168     GList *agents = NULL;
2169     GList *gIter2 = NULL;
2170 
2171     agents = resources_list_agents(class, provider);
2172     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2173         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2174             found = true;
2175         }
2176     }
2177     g_list_free_full(agents, free);
2178     return found;
2179 }
2180 
2181 static int
2182 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2183 {
2184     int rc = pcmk_ok;
2185     char *provider = NULL;
2186     GList *ocf_providers = NULL;
2187     GList *gIter = NULL;
2188 
2189     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2190 
2191     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2192         provider = gIter->data;
2193         if (!agent || does_provider_have_agent(agent, provider,
2194                                                PCMK_RESOURCE_CLASS_OCF)) {
2195             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2196             rc++;
2197         }
2198     }
2199 
2200     g_list_free_full(ocf_providers, free);
2201     return rc;
2202 }
2203 
2204 static int
2205 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2206 {
2207     int rc = 0;
2208     GList *standards = NULL;
2209     GList *gIter = NULL;
2210 
2211     standards = resources_list_standards();
2212 
2213     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2214         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2215         rc++;
2216     }
2217 
2218     if (list_stonith_agents(NULL) > 0) {
2219         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2220         rc++;
2221     }
2222 
2223     g_list_free_full(standards, free);
2224     return rc;
2225 }
2226 
2227 /*!
2228  * \internal
2229  * \brief Create an executor API object
2230  *
2231  * \param[out] api       Will be set to newly created API object (it is the
2232  *                       caller's responsibility to free this value with
2233  *                       lrmd_api_delete() if this function succeeds)
2234  * \param[in]  nodename  If the object will be used for a remote connection,
2235  *                       the node name to use in cluster for remote executor
2236  * \param[in]  server    If the object will be used for a remote connection,
2237  *                       the resolvable host name to connect to
2238  * \param[in]  port      If the object will be used for a remote connection,
2239  *                       port number on \p server to connect to
2240  *
2241  * \return Standard Pacemaker return code
2242  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2243  *       value will be used for both. If the caller leaves both NULL, an API
2244  *       object will be created for a local executor connection.
2245  */
2246 int
2247 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2248 {
2249     lrmd_private_t *pvt = NULL;
2250 
2251     if (api == NULL) {
2252         return EINVAL;
2253     }
2254     *api = NULL;
2255 
2256     // Allocate all memory needed
2257 
2258     *api = calloc(1, sizeof(lrmd_t));
2259     if (*api == NULL) {
2260         return ENOMEM;
2261     }
2262 
2263     pvt = calloc(1, sizeof(lrmd_private_t));
2264     if (pvt == NULL) {
2265         lrmd_api_delete(*api);
2266         *api = NULL;
2267         return ENOMEM;
2268     }
2269     (*api)->lrmd_private = pvt;
2270 
2271     // @TODO Do we need to do this for local connections?
2272     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2273 
2274     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2275 
2276     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2277         lrmd_api_delete(*api);
2278         *api = NULL;
2279         return ENOMEM;
2280     }
2281 
2282     // Set methods
2283     (*api)->cmds->connect = lrmd_api_connect;
2284     (*api)->cmds->connect_async = lrmd_api_connect_async;
2285     (*api)->cmds->is_connected = lrmd_api_is_connected;
2286     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2287     (*api)->cmds->disconnect = lrmd_api_disconnect;
2288     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2289     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2290     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2291     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2292     (*api)->cmds->set_callback = lrmd_api_set_callback;
2293     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2294     (*api)->cmds->exec = lrmd_api_exec;
2295     (*api)->cmds->cancel = lrmd_api_cancel;
2296     (*api)->cmds->list_agents = lrmd_api_list_agents;
2297     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2298     (*api)->cmds->list_standards = lrmd_api_list_standards;
2299     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2300     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2301 
2302     if ((nodename == NULL) && (server == NULL)) {
2303         pvt->type = pcmk__client_ipc;
2304     } else {
2305 #ifdef HAVE_GNUTLS_GNUTLS_H
2306         if (nodename == NULL) {
2307             nodename = server;
2308         } else if (server == NULL) {
2309             server = nodename;
2310         }
2311         pvt->type = pcmk__client_tls;
2312         pvt->remote_nodename = strdup(nodename);
2313         pvt->server = strdup(server);
2314         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2315             lrmd_api_delete(*api);
2316             *api = NULL;
2317             return ENOMEM;
2318         }
2319         pvt->port = port;
2320         if (pvt->port == 0) {
2321             pvt->port = crm_default_remote_port();
2322         }
2323 #else
2324         crm_err("Cannot communicate with Pacemaker Remote "
2325                 "because GnuTLS is not enabled for this build");
2326         lrmd_api_delete(*api);
2327         *api = NULL;
2328         return EOPNOTSUPP;
2329 #endif
2330     }
2331     return pcmk_rc_ok;
2332 }
2333 
2334 lrmd_t *
2335 lrmd_api_new(void)
     /*  */
2336 {
2337     lrmd_t *api = NULL;
2338 
2339     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2340     return api;
2341 }
2342 
2343 lrmd_t *
2344 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2345 {
2346     lrmd_t *api = NULL;
2347 
2348     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2349     return api;
2350 }
2351 
2352 void
2353 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2354 {
2355     if (lrmd == NULL) {
2356         return;
2357     }
2358     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2359         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2360             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2361         }
2362         free(lrmd->cmds);
2363     }
2364     if (lrmd->lrmd_private != NULL) {
2365         lrmd_private_t *native = lrmd->lrmd_private;
2366 
2367 #ifdef HAVE_GNUTLS_GNUTLS_H
2368         free(native->server);
2369 #endif
2370         free(native->remote_nodename);
2371         free(native->remote);
2372         free(native->token);
2373         free(native->peer_version);
2374         free(lrmd->lrmd_private);
2375     }
2376     free(lrmd);
2377 }
2378 
2379 struct metadata_cb {
2380      void (*callback)(int pid, const pcmk__action_result_t *result,
2381                       void *user_data);
2382      void *user_data;
2383 };
2384 
2385 /*!
2386  * \internal
2387  * \brief Process asynchronous metadata completion
2388  *
2389  * \param[in,out] action  Metadata action that completed
2390  */
2391 static void
2392 metadata_complete(svc_action_t *action)
     /*  */
2393 {
2394     struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2395     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2396 
2397     pcmk__set_result(&result, action->rc, action->status,
2398                      services__exit_reason(action));
2399     pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2400 
2401     metadata_cb->callback(0, &result, metadata_cb->user_data);
2402     result.action_stdout = NULL; // Prevent free, because action owns it
2403     result.action_stderr = NULL; // Prevent free, because action owns it
2404     pcmk__reset_result(&result);
2405     free(metadata_cb);
2406 }
2407 
2408 /*!
2409  * \internal
2410  * \brief Retrieve agent metadata asynchronously
2411  *
2412  * \param[in]     rsc        Resource agent specification
2413  * \param[in]     callback   Function to call with result (this will always be
2414  *                           called, whether by this function directly or later
2415  *                           via the main loop, and on success the metadata will
2416  *                           be in its result argument's action_stdout)
2417  * \param[in,out] user_data  User data to pass to callback
2418  *
2419  * \return Standard Pacemaker return code
2420  * \note This function is not a lrmd_api_operations_t method because it does not
2421  *       need an lrmd_t object and does not go through the executor, but
2422  *       executes the agent directly.
2423  */
2424 int
2425 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
     /*  */
2426                      void (*callback)(int pid,
2427                                       const pcmk__action_result_t *result,
2428                                       void *user_data),
2429                      void *user_data)
2430 {
2431     svc_action_t *action = NULL;
2432     struct metadata_cb *metadata_cb = NULL;
2433     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2434 
2435     CRM_CHECK(callback != NULL, return EINVAL);
2436 
2437     if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2438         pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2439                          PCMK_EXEC_ERROR_FATAL,
2440                          "Invalid resource specification");
2441         callback(0, &result, user_data);
2442         pcmk__reset_result(&result);
2443         return EINVAL;
2444     }
2445 
2446     if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2447         return stonith__metadata_async(rsc->type,
2448                                        PCMK_DEFAULT_METADATA_TIMEOUT_MS / 1000,
2449                                        callback, user_data);
2450     }
2451 
2452     action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2453                                               rsc->standard, rsc->provider,
2454                                               rsc->type,
2455                                               PCMK_ACTION_META_DATA, 0,
2456                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
2457                                               NULL, 0);
2458     if (action == NULL) {
2459         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2460                          "Out of memory");
2461         callback(0, &result, user_data);
2462         pcmk__reset_result(&result);
2463         return ENOMEM;
2464     }
2465     if (action->rc != PCMK_OCF_UNKNOWN) {
2466         pcmk__set_result(&result, action->rc, action->status,
2467                          services__exit_reason(action));
2468         callback(0, &result, user_data);
2469         pcmk__reset_result(&result);
2470         services_action_free(action);
2471         return EINVAL;
2472     }
2473 
2474     action->cb_data = calloc(1, sizeof(struct metadata_cb));
2475     if (action->cb_data == NULL) {
2476         services_action_free(action);
2477         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2478                          "Out of memory");
2479         callback(0, &result, user_data);
2480         pcmk__reset_result(&result);
2481         return ENOMEM;
2482     }
2483 
2484     metadata_cb = (struct metadata_cb *) action->cb_data;
2485     metadata_cb->callback = callback;
2486     metadata_cb->user_data = user_data;
2487     if (!services_action_async(action, metadata_complete)) {
2488         services_action_free(action);
2489         return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2490     }
2491 
2492     // The services library has taken responsibility for action
2493     return pcmk_rc_ok;
2494 }
2495 
2496 /*!
2497  * \internal
2498  * \brief Set the result of an executor event
2499  *
2500  * \param[in,out] event        Executor event to set
2501  * \param[in]     rc           OCF exit status of event
2502  * \param[in]     op_status    Executor status of event
2503  * \param[in]     exit_reason  Human-friendly description of event
2504  */
2505 void
2506 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2507                  const char *exit_reason)
2508 {
2509     if (event == NULL) {
2510         return;
2511     }
2512 
2513     event->rc = rc;
2514     event->op_status = op_status;
2515     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2516 }
2517 
2518 /*!
2519  * \internal
2520  * \brief Clear an executor event's exit reason, output, and error output
2521  *
2522  * \param[in,out] event  Executor event to reset
2523  */
2524 void
2525 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2526 {
2527     if (event == NULL) {
2528         return;
2529     }
2530 
2531     free((void *) event->exit_reason);
2532     event->exit_reason = NULL;
2533 
2534     free((void *) event->output);
2535     event->output = NULL;
2536 }
2537 
2538 /*!
2539  * \internal
2540  * \brief Get the uptime of a remote resource connection
2541  *
2542  * When the cluster connects to a remote resource, part of that resource's
2543  * handshake includes the uptime of the remote resource's connection.  This
2544  * uptime is stored in the lrmd_t object.
2545  *
2546  * \return The connection's uptime, or -1 if unknown
2547  */
2548 time_t
2549 lrmd__uptime(lrmd_t *lrmd)
     /*  */
2550 {
2551     lrmd_private_t *native = lrmd->lrmd_private;
2552 
2553     if (native->remote == NULL) {
2554         return -1;
2555     } else {
2556         return native->remote->uptime;
2557     }
2558 }
2559 
2560 const char *
2561 lrmd__node_start_state(lrmd_t *lrmd)
     /*  */
2562 {
2563     lrmd_private_t *native = lrmd->lrmd_private;
2564 
2565     if (native->remote == NULL) {
2566         return NULL;
2567     } else {
2568         return native->remote->start_state;
2569     }
2570 }