root/lib/lrmd/lrmd_client.c

/* */

DEFINITIONS

1. lrmd_list_add
2. lrmd_list_freeall
3. lrmd_key_value_add
4. lrmd_key_value_freeall
5. lrmd_new_event
6. lrmd_copy_event
7. lrmd_free_event
8. lrmd_dispatch_internal
9. lrmd_ipc_dispatch
10. lrmd_free_xml
11. remote_executor_connected
12. lrmd_tls_dispatch
13. lrmd_poll
14. lrmd_dispatch
15. lrmd_create_op
16. lrmd_ipc_connection_destroy
17. lrmd_tls_connection_destroy
18. lrmd__remote_send_xml
19. read_remote_reply
20. send_remote_message
21. lrmd_tls_send_recv
22. lrmd_send_xml
23. lrmd_send_xml_no_reply
24. lrmd_api_is_connected
25. lrmd_send_command
26. lrmd_api_poke_connection
27. lrmd__validate_remote_settings
28. lrmd_handshake
29. lrmd_ipc_connect
30. copy_gnutls_datum
31. clear_gnutls_datum
32. read_gnutls_key
33. key_is_cached
34. key_cache_expired
35. clear_key_cache
36. get_cached_key
37. cache_key
38. get_remote_key
39. lrmd__init_remote_key
40. lrmd_gnutls_global_init
41. report_async_connection_result
42. lrmd__tls_client_handshake
43. add_tls_to_mainloop
44. lrmd_tcp_connect_cb
45. lrmd_tls_connect_async
46. lrmd_tls_connect
47. lrmd_api_connect
48. lrmd_api_connect_async
49. lrmd_ipc_disconnect
50. lrmd_tls_disconnect
51. lrmd_api_disconnect
52. lrmd_api_register_rsc
53. lrmd_api_unregister_rsc
54. lrmd_new_rsc_info
55. lrmd_copy_rsc_info
56. lrmd_free_rsc_info
57. lrmd_api_get_rsc_info
58. lrmd_free_op_info
59. lrmd_api_get_recurring_ops
60. lrmd_api_set_callback
61. lrmd_internal_set_proxy_callback
62. lrmd_internal_proxy_dispatch
63. lrmd_internal_proxy_send
64. stonith_get_metadata
65. lrmd_api_get_metadata
66. lrmd_api_get_metadata_params
67. lrmd_api_exec
68. lrmd_api_exec_alert
69. lrmd_api_cancel
70. list_stonith_agents
71. lrmd_api_list_agents
72. does_provider_have_agent
73. lrmd_api_list_ocf_providers
74. lrmd_api_list_standards
75. lrmd__new
76. lrmd_api_new
77. lrmd_remote_api_new
78. lrmd_api_delete
79. lrmd__set_result
80. lrmd__reset_result

   1 /*
   2  * Copyright 2012-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/msg_xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  include <gnutls/gnutls.h>
  42 #endif
  43 
  44 #include <sys/socket.h>
  45 #include <netinet/in.h>
  46 #include <netinet/ip.h>
  47 #include <arpa/inet.h>
  48 #include <netdb.h>
  49 
  50 #define MAX_TLS_RECV_WAIT 10000
  51 
  52 CRM_TRACE_INIT_DATA(lrmd);
  53 
  54 static int lrmd_api_disconnect(lrmd_t * lrmd);
  55 static int lrmd_api_is_connected(lrmd_t * lrmd);
  56 
  57 /* IPC proxy functions */
  58 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  59 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  60 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  61 
  62 #ifdef HAVE_GNUTLS_GNUTLS_H
  63 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  64 gnutls_psk_client_credentials_t psk_cred_s;
  65 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  66 static int global_remote_msg_id = 0;
  67 static void lrmd_tls_connection_destroy(gpointer userdata);
  68 #endif
  69 
  70 typedef struct lrmd_private_s {
  71     uint64_t type;
  72     char *token;
  73     mainloop_io_t *source;
  74 
  75     /* IPC parameters */
  76     crm_ipc_t *ipc;
  77 
  78     pcmk__remote_t *remote;
  79 
  80     /* Extra TLS parameters */
  81     char *remote_nodename;
  82 #ifdef HAVE_GNUTLS_GNUTLS_H
  83     char *server;
  84     int port;
  85     gnutls_psk_client_credentials_t psk_cred_c;
  86 
  87     /* while the async connection is occurring, this is the id
  88      * of the connection timeout timer. */
  89     int async_timer;
  90     int sock;
  91     /* since tls requires a round trip across the network for a
  92      * request/reply, there are times where we just want to be able
  93      * to send a request from the client and not wait around (or even care
  94      * about) what the reply is. */
  95     int expected_late_replies;
  96     GList *pending_notify;
  97     crm_trigger_t *process_notify;
  98 #endif
  99 
 100     lrmd_event_callback callback;
 101 
 102     /* Internal IPC proxy msg passing for remote guests */
 103     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 104     void *proxy_callback_userdata;
 105     char *peer_version;
 106 } lrmd_private_t;
 107 
 108 static lrmd_list_t *
 109 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 110 {
 111     lrmd_list_t *p, *end;
 112 
 113     p = calloc(1, sizeof(lrmd_list_t));
 114     p->val = strdup(value);
 115 
 116     end = head;
 117     while (end && end->next) {
 118         end = end->next;
 119     }
 120 
 121     if (end) {
 122         end->next = p;
 123     } else {
 124         head = p;
 125     }
 126 
 127     return head;
 128 }
 129 
 130 void
 131 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 132 {
 133     lrmd_list_t *p;
 134 
 135     while (head) {
 136         char *val = (char *)head->val;
 137 
 138         p = head->next;
 139         free(val);
 140         free(head);
 141         head = p;
 142     }
 143 }
 144 
 145 lrmd_key_value_t *
 146 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 147 {
 148     lrmd_key_value_t *p, *end;
 149 
 150     p = calloc(1, sizeof(lrmd_key_value_t));
 151     p->key = strdup(key);
 152     p->value = strdup(value);
 153 
 154     end = head;
 155     while (end && end->next) {
 156         end = end->next;
 157     }
 158 
 159     if (end) {
 160         end->next = p;
 161     } else {
 162         head = p;
 163     }
 164 
 165     return head;
 166 }
 167 
 168 void
 169 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 170 {
 171     lrmd_key_value_t *p;
 172 
 173     while (head) {
 174         p = head->next;
 175         free(head->key);
 176         free(head->value);
 177         free(head);
 178         head = p;
 179     }
 180 }
 181 
 182 /*!
 183  * Create a new lrmd_event_data_t object
 184  *
 185  * \param[in] rsc_id       ID of resource involved in event
 186  * \param[in] task         Action name
 187  * \param[in] interval_ms  Action interval
 188  *
 189  * \return Newly allocated and initialized lrmd_event_data_t
 190  * \note This functions asserts on memory errors, so the return value is
 191  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 192  *       result with lrmd_free_event().
 193  */
 194 lrmd_event_data_t *
 195 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 196 {
 197     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 198 
 199     CRM_ASSERT(event != NULL);
 200     pcmk__str_update((char **) &event->rsc_id, rsc_id);
 201     pcmk__str_update((char **) &event->op_type, task);
 202     event->interval_ms = interval_ms;
 203     return event;
 204 }
 205 
 206 lrmd_event_data_t *
 207 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 208 {
 209     lrmd_event_data_t *copy = NULL;
 210 
 211     copy = calloc(1, sizeof(lrmd_event_data_t));
 212 
 213     copy->type = event->type;
 214     pcmk__str_update((char **) &copy->rsc_id, event->rsc_id);
 215     pcmk__str_update((char **) &copy->op_type, event->op_type);
 216     pcmk__str_update((char **) &copy->user_data, event->user_data);
 217     copy->call_id = event->call_id;
 218     copy->timeout = event->timeout;
 219     copy->interval_ms = event->interval_ms;
 220     copy->start_delay = event->start_delay;
 221     copy->rsc_deleted = event->rsc_deleted;
 222     copy->rc = event->rc;
 223     copy->op_status = event->op_status;
 224     pcmk__str_update((char **) &copy->output, event->output);
 225     copy->t_run = event->t_run;
 226     copy->t_rcchange = event->t_rcchange;
 227     copy->exec_time = event->exec_time;
 228     copy->queue_time = event->queue_time;
 229     copy->connection_rc = event->connection_rc;
 230     copy->params = pcmk__str_table_dup(event->params);
 231     pcmk__str_update((char **) &copy->remote_nodename, event->remote_nodename);
 232     pcmk__str_update((char **) &copy->exit_reason, event->exit_reason);
 233 
 234     return copy;
 235 }
 236 
 237 /*!
 238  * \brief Free an executor event
 239  *
 240  * \param[in]  Executor event object to free
 241  */
 242 void
 243 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 244 {
 245     if (event == NULL) {
 246         return;
 247     }
 248     // @TODO Why are these const char *?
 249     free((void *) event->rsc_id);
 250     free((void *) event->op_type);
 251     free((void *) event->user_data);
 252     free((void *) event->remote_nodename);
 253     lrmd__reset_result(event);
 254     if (event->params != NULL) {
 255         g_hash_table_destroy(event->params);
 256     }
 257     free(event);
 258 }
 259 
 260 static void
 261 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 262 {
 263     const char *type;
 264     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 265     lrmd_private_t *native = lrmd->lrmd_private;
 266     lrmd_event_data_t event = { 0, };
 267 
 268     if (proxy_session != NULL) {
 269         /* this is proxy business */
 270         lrmd_internal_proxy_dispatch(lrmd, msg);
 271         return;
 272     } else if (!native->callback) {
 273         /* no callback set */
 274         crm_trace("notify event received but client has not set callback");
 275         return;
 276     }
 277 
 278     event.remote_nodename = native->remote_nodename;
 279     type = crm_element_value(msg, F_LRMD_OPERATION);
 280     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 281     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 282 
 283     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 284         event.type = lrmd_event_register;
 285     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 286         event.type = lrmd_event_unregister;
 287     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 288         time_t epoch = 0;
 289 
 290         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 291         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 292         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 293         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 294         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 295         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 296 
 297         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 298         event.t_run = (unsigned int) epoch;
 299 
 300         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 301         event.t_rcchange = (unsigned int) epoch;
 302 
 303         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 304         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 305 
 306         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 307         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 308         event.type = lrmd_event_exec_complete;
 309 
 310         /* output and exit_reason may be freed by a callback */
 311         event.output = crm_element_value_copy(msg, F_LRMD_RSC_OUTPUT);
 312         lrmd__set_result(&event, event.rc, event.op_status,
 313                          crm_element_value(msg, F_LRMD_RSC_EXIT_REASON));
 314 
 315         event.params = xml2list(msg);
 316     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 317         event.type = lrmd_event_new_client;
 318     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 319         event.type = lrmd_event_poke;
 320     } else {
 321         return;
 322     }
 323 
 324     crm_trace("op %s notify event received", type);
 325     native->callback(&event);
 326 
 327     if (event.params) {
 328         g_hash_table_destroy(event.params);
 329     }
 330     lrmd__reset_result(&event);
 331 }
 332 
 333 // \return Always 0, to indicate that IPC mainloop source should be kept
 334 static int
 335 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 336 {
 337     lrmd_t *lrmd = userdata;
 338     lrmd_private_t *native = lrmd->lrmd_private;
 339 
 340     if (native->callback != NULL) {
 341         xmlNode *msg = string2xml(buffer);
 342 
 343         lrmd_dispatch_internal(lrmd, msg);
 344         free_xml(msg);
 345     }
 346     return 0;
 347 }
 348 
 349 #ifdef HAVE_GNUTLS_GNUTLS_H
 350 static void
 351 lrmd_free_xml(gpointer userdata)
     /*  */
 352 {
 353     free_xml((xmlNode *) userdata);
 354 }
 355 
 356 static bool
 357 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 358 {
 359     lrmd_private_t *native = lrmd->lrmd_private;
 360 
 361     return (native->remote->tls_session != NULL);
 362 }
 363 
 364 /*!
 365  * \internal
 366  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 367  *
 368  * \param[in] userdata  API connection
 369  *
 370  * \return Always return a nonnegative value, which as a file descriptor
 371  *         dispatch function means keep the mainloop source, and as a
 372  *         trigger dispatch function, 0 means remove the trigger from the
 373  *         mainloop while 1 means keep it (and job completed)
 374  */
 375 static int
 376 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 377 {
 378     lrmd_t *lrmd = userdata;
 379     lrmd_private_t *native = lrmd->lrmd_private;
 380     xmlNode *xml = NULL;
 381     int rc = pcmk_rc_ok;
 382 
 383     if (!remote_executor_connected(lrmd)) {
 384         crm_trace("TLS dispatch triggered after disconnect");
 385         return 0;
 386     }
 387 
 388     crm_trace("TLS dispatch triggered");
 389 
 390     /* First check if there are any pending notifies to process that came
 391      * while we were waiting for replies earlier. */
 392     if (native->pending_notify) {
 393         GList *iter = NULL;
 394 
 395         crm_trace("Processing pending notifies");
 396         for (iter = native->pending_notify; iter; iter = iter->next) {
 397             lrmd_dispatch_internal(lrmd, iter->data);
 398         }
 399         g_list_free_full(native->pending_notify, lrmd_free_xml);
 400         native->pending_notify = NULL;
 401     }
 402 
 403     /* Next read the current buffer and see if there are any messages to handle. */
 404     switch (pcmk__remote_ready(native->remote, 0)) {
 405         case pcmk_rc_ok:
 406             rc = pcmk__read_remote_message(native->remote, -1);
 407             xml = pcmk__remote_message_xml(native->remote);
 408             break;
 409         case ETIME:
 410             // Nothing to read, check if a full message is already in buffer
 411             xml = pcmk__remote_message_xml(native->remote);
 412             break;
 413         default:
 414             rc = ENOTCONN;
 415             break;
 416     }
 417     while (xml) {
 418         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 419         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 420             lrmd_dispatch_internal(lrmd, xml);
 421         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 422             if (native->expected_late_replies > 0) {
 423                 native->expected_late_replies--;
 424             } else {
 425                 int reply_id = 0;
 426                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 427                 /* if this happens, we want to know about it */
 428                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 429             }
 430         }
 431         free_xml(xml);
 432         xml = pcmk__remote_message_xml(native->remote);
 433     }
 434 
 435     if (rc == ENOTCONN) {
 436         crm_info("Lost %s executor connection while reading data",
 437                  (native->remote_nodename? native->remote_nodename : "local"));
 438         lrmd_tls_disconnect(lrmd);
 439         return 0;
 440     }
 441     return 1;
 442 }
 443 #endif
 444 
 445 /* Not used with mainloop */
 446 int
 447 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 448 {
 449     lrmd_private_t *native = lrmd->lrmd_private;
 450 
 451     switch (native->type) {
 452         case pcmk__client_ipc:
 453             return crm_ipc_ready(native->ipc);
 454 
 455 #ifdef HAVE_GNUTLS_GNUTLS_H
 456         case pcmk__client_tls:
 457             if (native->pending_notify) {
 458                 return 1;
 459             } else {
 460                 int rc = pcmk__remote_ready(native->remote, 0);
 461 
 462                 switch (rc) {
 463                     case pcmk_rc_ok:
 464                         return 1;
 465                     case ETIME:
 466                         return 0;
 467                     default:
 468                         return pcmk_rc2legacy(rc);
 469                 }
 470             }
 471 #endif
 472         default:
 473             crm_err("Unsupported executor connection type (bug?): %d",
 474                     native->type);
 475             return -EPROTONOSUPPORT;
 476     }
 477 }
 478 
 479 /* Not used with mainloop */
 480 bool
 481 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 482 {
 483     lrmd_private_t *private = NULL;
 484 
 485     CRM_ASSERT(lrmd != NULL);
 486 
 487     private = lrmd->lrmd_private;
 488     switch (private->type) {
 489         case pcmk__client_ipc:
 490             while (crm_ipc_ready(private->ipc)) {
 491                 if (crm_ipc_read(private->ipc) > 0) {
 492                     const char *msg = crm_ipc_buffer(private->ipc);
 493 
 494                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 495                 }
 496             }
 497             break;
 498 #ifdef HAVE_GNUTLS_GNUTLS_H
 499         case pcmk__client_tls:
 500             lrmd_tls_dispatch(lrmd);
 501             break;
 502 #endif
 503         default:
 504             crm_err("Unsupported executor connection type (bug?): %d",
 505                     private->type);
 506     }
 507 
 508     if (lrmd_api_is_connected(lrmd) == FALSE) {
 509         crm_err("Connection closed");
 510         return FALSE;
 511     }
 512 
 513     return TRUE;
 514 }
 515 
 516 static xmlNode *
 517 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 518                enum lrmd_call_options options)
 519 {
 520     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 521 
 522     CRM_CHECK(op_msg != NULL, return NULL);
 523     CRM_CHECK(token != NULL, return NULL);
 524 
 525     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 526     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 527     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 528     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 529     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 530     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 531 
 532     if (data != NULL) {
 533         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 534     }
 535 
 536     crm_trace("Created executor %s command with call options %.8lx (%d)",
 537               op, (long)options, options);
 538     return op_msg;
 539 }
 540 
 541 static void
 542 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 543 {
 544     lrmd_t *lrmd = userdata;
 545     lrmd_private_t *native = lrmd->lrmd_private;
 546 
 547     crm_info("IPC connection destroyed");
 548 
 549     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 550     native->ipc = NULL;
 551     native->source = NULL;
 552 
 553     if (native->callback) {
 554         lrmd_event_data_t event = { 0, };
 555         event.type = lrmd_event_disconnect;
 556         event.remote_nodename = native->remote_nodename;
 557         native->callback(&event);
 558     }
 559 }
 560 
 561 #ifdef HAVE_GNUTLS_GNUTLS_H
 562 static void
 563 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 564 {
 565     lrmd_t *lrmd = userdata;
 566     lrmd_private_t *native = lrmd->lrmd_private;
 567 
 568     crm_info("TLS connection destroyed");
 569 
 570     if (native->remote->tls_session) {
 571         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 572         gnutls_deinit(*native->remote->tls_session);
 573         gnutls_free(native->remote->tls_session);
 574     }
 575     if (native->psk_cred_c) {
 576         gnutls_psk_free_client_credentials(native->psk_cred_c);
 577     }
 578     if (native->sock) {
 579         close(native->sock);
 580     }
 581     if (native->process_notify) {
 582         mainloop_destroy_trigger(native->process_notify);
 583         native->process_notify = NULL;
 584     }
 585     if (native->pending_notify) {
 586         g_list_free_full(native->pending_notify, lrmd_free_xml);
 587         native->pending_notify = NULL;
 588     }
 589 
 590     free(native->remote->buffer);
 591     native->remote->buffer = NULL;
 592     native->source = 0;
 593     native->sock = 0;
 594     native->psk_cred_c = NULL;
 595     native->remote->tls_session = NULL;
 596     native->sock = 0;
 597 
 598     if (native->callback) {
 599         lrmd_event_data_t event = { 0, };
 600         event.remote_nodename = native->remote_nodename;
 601         event.type = lrmd_event_disconnect;
 602         native->callback(&event);
 603     }
 604     return;
 605 }
 606 
 607 // \return Standard Pacemaker return code
 608 int
 609 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 610                       const char *msg_type)
 611 {
 612     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 613     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 614     return pcmk__remote_send_xml(session, msg);
 615 }
 616 
 617 // \return Standard Pacemaker return code
 618 static int
 619 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 620                   xmlNode **reply)
 621 {
 622     lrmd_private_t *native = lrmd->lrmd_private;
 623     time_t start = time(NULL);
 624     const char *msg_type = NULL;
 625     int reply_id = 0;
 626     int remaining_timeout = 0;
 627     int rc = pcmk_rc_ok;
 628 
 629     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 630      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 631     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 632         total_timeout = MAX_TLS_RECV_WAIT;
 633     }
 634 
 635     for (*reply = NULL; *reply == NULL; ) {
 636 
 637         *reply = pcmk__remote_message_xml(native->remote);
 638         if (*reply == NULL) {
 639             /* read some more off the tls buffer if we still have time left. */
 640             if (remaining_timeout) {
 641                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 642             } else {
 643                 remaining_timeout = total_timeout;
 644             }
 645             if (remaining_timeout <= 0) {
 646                 return ETIME;
 647             }
 648 
 649             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 650             if (rc != pcmk_rc_ok) {
 651                 return rc;
 652             }
 653 
 654             *reply = pcmk__remote_message_xml(native->remote);
 655             if (*reply == NULL) {
 656                 return ENOMSG;
 657             }
 658         }
 659 
 660         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
 661         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 662 
 663         if (!msg_type) {
 664             crm_err("Empty msg type received while waiting for reply");
 665             free_xml(*reply);
 666             *reply = NULL;
 667         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 668             /* got a notify while waiting for reply, trigger the notify to be processed later */
 669             crm_info("queueing notify");
 670             native->pending_notify = g_list_append(native->pending_notify, *reply);
 671             if (native->process_notify) {
 672                 crm_info("notify trigger set.");
 673                 mainloop_set_trigger(native->process_notify);
 674             }
 675             *reply = NULL;
 676         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 677             /* msg isn't a reply, make some noise */
 678             crm_err("Expected a reply, got %s", msg_type);
 679             free_xml(*reply);
 680             *reply = NULL;
 681         } else if (reply_id != expected_reply_id) {
 682             if (native->expected_late_replies > 0) {
 683                 native->expected_late_replies--;
 684             } else {
 685                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 686             }
 687             free_xml(*reply);
 688             *reply = NULL;
 689         }
 690     }
 691 
 692     if (native->remote->buffer && native->process_notify) {
 693         mainloop_set_trigger(native->process_notify);
 694     }
 695 
 696     return rc;
 697 }
 698 
 699 // \return Standard Pacemaker return code
 700 static int
 701 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 702 {
 703     int rc = pcmk_rc_ok;
 704     lrmd_private_t *native = lrmd->lrmd_private;
 705 
 706     global_remote_msg_id++;
 707     if (global_remote_msg_id <= 0) {
 708         global_remote_msg_id = 1;
 709     }
 710 
 711     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 712                                "request");
 713     if (rc != pcmk_rc_ok) {
 714         crm_err("Disconnecting because TLS message could not be sent to "
 715                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 716         lrmd_tls_disconnect(lrmd);
 717     }
 718     return rc;
 719 }
 720 
 721 static int
 722 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 723 {
 724     int rc = 0;
 725     xmlNode *xml = NULL;
 726 
 727     if (!remote_executor_connected(lrmd)) {
 728         return -ENOTCONN;
 729     }
 730 
 731     rc = send_remote_message(lrmd, msg);
 732     if (rc != pcmk_rc_ok) {
 733         return pcmk_rc2legacy(rc);
 734     }
 735 
 736     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 737     if (rc != pcmk_rc_ok) {
 738         crm_err("Disconnecting remote after request %d reply not received: %s "
 739                 CRM_XS " rc=%d timeout=%dms",
 740                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 741         lrmd_tls_disconnect(lrmd);
 742     }
 743 
 744     if (reply) {
 745         *reply = xml;
 746     } else {
 747         free_xml(xml);
 748     }
 749 
 750     return pcmk_rc2legacy(rc);
 751 }
 752 #endif
 753 
 754 static int
 755 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 756 {
 757     int rc = pcmk_ok;
 758     lrmd_private_t *native = lrmd->lrmd_private;
 759 
 760     switch (native->type) {
 761         case pcmk__client_ipc:
 762             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 763             break;
 764 #ifdef HAVE_GNUTLS_GNUTLS_H
 765         case pcmk__client_tls:
 766             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 767             break;
 768 #endif
 769         default:
 770             crm_err("Unsupported executor connection type (bug?): %d",
 771                     native->type);
 772             rc = -EPROTONOSUPPORT;
 773     }
 774 
 775     return rc;
 776 }
 777 
 778 static int
 779 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 780 {
 781     int rc = pcmk_ok;
 782     lrmd_private_t *native = lrmd->lrmd_private;
 783 
 784     switch (native->type) {
 785         case pcmk__client_ipc:
 786             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 787             break;
 788 #ifdef HAVE_GNUTLS_GNUTLS_H
 789         case pcmk__client_tls:
 790             rc = send_remote_message(lrmd, msg);
 791             if (rc == pcmk_rc_ok) {
 792                 /* we don't want to wait around for the reply, but
 793                  * since the request/reply protocol needs to behave the same
 794                  * as libqb, a reply will eventually come later anyway. */
 795                 native->expected_late_replies++;
 796             }
 797             rc = pcmk_rc2legacy(rc);
 798             break;
 799 #endif
 800         default:
 801             crm_err("Unsupported executor connection type (bug?): %d",
 802                     native->type);
 803             rc = -EPROTONOSUPPORT;
 804     }
 805 
 806     return rc;
 807 }
 808 
 809 static int
 810 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 811 {
 812     lrmd_private_t *native = lrmd->lrmd_private;
 813 
 814     switch (native->type) {
 815         case pcmk__client_ipc:
 816             return crm_ipc_connected(native->ipc);
 817 #ifdef HAVE_GNUTLS_GNUTLS_H
 818         case pcmk__client_tls:
 819             return remote_executor_connected(lrmd);
 820 #endif
 821         default:
 822             crm_err("Unsupported executor connection type (bug?): %d",
 823                     native->type);
 824             return 0;
 825     }
 826 }
 827 
 828 /*!
 829  * \internal
 830  * \brief Send a prepared API command to the executor
 831  *
 832  * \param[in]  lrmd          Existing connection to the executor
 833  * \param[in]  op            Name of API command to send
 834  * \param[in]  data          Command data XML to add to the sent command
 835  * \param[out] output_data   If expecting a reply, it will be stored here
 836  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to
 837  *                           a sensible value per the type of connection,
 838  *                           standard vs. pacemaker remote);
 839  *                           also propagated to the command XML
 840  * \param[in]  call_options  Call options to pass to server when sending
 841  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
 842  *                           must be TRUE for IPC (as opposed to TLS) clients
 843  *
 844  * \return pcmk_ok on success, -errno on error
 845  */
 846 static int
 847 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 848                   xmlNode **output_data, int timeout,
 849                   enum lrmd_call_options options, gboolean expect_reply)
 850 {
 851     int rc = pcmk_ok;
 852     lrmd_private_t *native = lrmd->lrmd_private;
 853     xmlNode *op_msg = NULL;
 854     xmlNode *op_reply = NULL;
 855 
 856     if (!lrmd_api_is_connected(lrmd)) {
 857         return -ENOTCONN;
 858     }
 859 
 860     if (op == NULL) {
 861         crm_err("No operation specified");
 862         return -EINVAL;
 863     }
 864 
 865     CRM_CHECK(native->token != NULL,;
 866         );
 867     crm_trace("Sending %s op to executor", op);
 868 
 869     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 870 
 871     if (op_msg == NULL) {
 872         return -EINVAL;
 873     }
 874 
 875     if (expect_reply) {
 876         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 877     } else {
 878         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 879         goto done;
 880     }
 881 
 882     if (rc < 0) {
 883         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 884         goto done;
 885 
 886     } else if(op_reply == NULL) {
 887         rc = -ENOMSG;
 888         goto done;
 889     }
 890 
 891     rc = pcmk_ok;
 892     crm_trace("%s op reply received", op);
 893     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 894         rc = -ENOMSG;
 895         goto done;
 896     }
 897 
 898     crm_log_xml_trace(op_reply, "Reply");
 899 
 900     if (output_data) {
 901         *output_data = op_reply;
 902         op_reply = NULL;        /* Prevent subsequent free */
 903     }
 904 
 905   done:
 906     if (lrmd_api_is_connected(lrmd) == FALSE) {
 907         crm_err("Executor disconnected");
 908     }
 909 
 910     free_xml(op_msg);
 911     free_xml(op_reply);
 912     return rc;
 913 }
 914 
 915 static int
 916 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 917 {
 918     int rc;
 919     lrmd_private_t *native = lrmd->lrmd_private;
 920     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 921 
 922     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 923     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 924                            (native->type == pcmk__client_ipc));
 925     free_xml(data);
 926 
 927     return rc < 0 ? rc : pcmk_ok;
 928 }
 929 
 930 // \return Standard Pacemaker return code
 931 int
 932 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 933 {
 934     int rc = pcmk_rc_ok;
 935     const char *value;
 936     lrmd_private_t *native = lrmd->lrmd_private;
 937     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 938 
 939     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 940 
 941     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 942     if ((value) &&
 943         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 944        crm_xml_add(data, F_LRMD_WATCHDOG, value);
 945     }
 946 
 947     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 948                            (native->type == pcmk__client_ipc));
 949     free_xml(data);
 950     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 951 }
 952 
 953 static int
 954 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 955 {
 956     int rc = pcmk_ok;
 957     lrmd_private_t *native = lrmd->lrmd_private;
 958     xmlNode *reply = NULL;
 959     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 960 
 961     crm_xml_add(hello, F_TYPE, T_LRMD);
 962     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 963     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 964     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 965 
 966     /* advertise that we are a proxy provider */
 967     if (native->proxy_callback) {
 968         pcmk__xe_set_bool_attr(hello, F_LRMD_IS_IPC_PROVIDER, true);
 969     }
 970 
 971     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 972 
 973     if (rc < 0) {
 974         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 975         rc = -ECOMM;
 976     } else if (reply == NULL) {
 977         crm_err("Did not receive registration reply");
 978         rc = -EPROTO;
 979     } else {
 980         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 981         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 982         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 983 
 984         crm_element_value_int(reply, F_LRMD_RC, &rc);
 985 
 986         if (rc == -EPROTO) {
 987             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
 988                 LRMD_PROTOCOL_VERSION, version);
 989             crm_log_xml_err(reply, "Protocol Error");
 990 
 991         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
 992             crm_err("Invalid registration message: %s", msg_type);
 993             crm_log_xml_err(reply, "Bad reply");
 994             rc = -EPROTO;
 995         } else if (tmp_ticket == NULL) {
 996             crm_err("No registration token provided");
 997             crm_log_xml_err(reply, "Bad reply");
 998             rc = -EPROTO;
 999         } else {
1000             crm_trace("Obtained registration token: %s", tmp_ticket);
1001             native->token = strdup(tmp_ticket);
1002             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1003             rc = pcmk_ok;
1004         }
1005     }
1006 
1007     free_xml(reply);
1008     free_xml(hello);
1009 
1010     if (rc != pcmk_ok) {
1011         lrmd_api_disconnect(lrmd);
1012     }
1013     return rc;
1014 }
1015 
1016 static int
1017 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1018 {
1019     int rc = pcmk_ok;
1020     lrmd_private_t *native = lrmd->lrmd_private;
1021 
1022     struct ipc_client_callbacks lrmd_callbacks = {
1023         .dispatch = lrmd_ipc_dispatch,
1024         .destroy = lrmd_ipc_connection_destroy
1025     };
1026 
1027     crm_info("Connecting to executor");
1028 
1029     if (fd) {
1030         /* No mainloop */
1031         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1032         if (native->ipc && crm_ipc_connect(native->ipc)) {
1033             *fd = crm_ipc_get_fd(native->ipc);
1034         } else if (native->ipc) {
1035             crm_perror(LOG_ERR, "Connection to executor failed");
1036             rc = -ENOTCONN;
1037         }
1038     } else {
1039         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1040         native->ipc = mainloop_get_ipc_client(native->source);
1041     }
1042 
1043     if (native->ipc == NULL) {
1044         crm_debug("Could not connect to the executor API");
1045         rc = -ENOTCONN;
1046     }
1047 
1048     return rc;
1049 }
1050 
1051 #ifdef HAVE_GNUTLS_GNUTLS_H
1052 static void
1053 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1054 {
1055     CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
1056 
1057     dest->data = gnutls_malloc(source->size);
1058     CRM_ASSERT(dest->data);
1059 
1060     memcpy(dest->data, source->data, source->size);
1061     dest->size = source->size;
1062 }
1063 
1064 static void
1065 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1066 {
1067     gnutls_free(datum->data);
1068     datum->data = NULL;
1069     datum->size = 0;
1070 }
1071 
1072 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1073 
1074 // \return Standard Pacemaker return code
1075 static int
1076 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1077 {
1078     FILE *stream = NULL;
1079     size_t buf_len = KEY_READ_LEN;
1080 
1081     if ((location == NULL) || (key == NULL)) {
1082         return EINVAL;
1083     }
1084 
1085     stream = fopen(location, "r");
1086     if (stream == NULL) {
1087         return errno;
1088     }
1089 
1090     key->data = gnutls_malloc(buf_len);
1091     key->size = 0;
1092     while (!feof(stream)) {
1093         int next = fgetc(stream);
1094 
1095         if (next == EOF) {
1096             if (!feof(stream)) {
1097                 crm_warn("Pacemaker Remote key read was partially successful "
1098                          "(copy in memory may be corrupted)");
1099             }
1100             break;
1101         }
1102         if (key->size == buf_len) {
1103             buf_len = key->size + KEY_READ_LEN;
1104             key->data = gnutls_realloc(key->data, buf_len);
1105             CRM_ASSERT(key->data);
1106         }
1107         key->data[key->size++] = (unsigned char) next;
1108     }
1109     fclose(stream);
1110 
1111     if (key->size == 0) {
1112         clear_gnutls_datum(key);
1113         return ENOKEY;
1114     }
1115     return pcmk_rc_ok;
1116 }
1117 
1118 // Cache the most recently used Pacemaker Remote authentication key
1119 
1120 struct key_cache_s {
1121     time_t updated;         // When cached key was read (valid for 1 minute)
1122     const char *location;   // Where cached key was read from
1123     gnutls_datum_t key;     // Cached key
1124 };
1125 
1126 static bool
1127 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1128 {
1129     return key_cache->updated != 0;
1130 }
1131 
1132 static bool
1133 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1134 {
1135     return (time(NULL) - key_cache->updated) >= 60;
1136 }
1137 
1138 static void
1139 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1140 {
1141     clear_gnutls_datum(&(key_cache->key));
1142     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1143         key_cache->updated = 0;
1144         key_cache->location = NULL;
1145         crm_debug("Cleared Pacemaker Remote key cache");
1146     }
1147 }
1148 
1149 static void
1150 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1151 {
1152     copy_gnutls_datum(key, &(key_cache->key));
1153     crm_debug("Using cached Pacemaker Remote key from %s",
1154               crm_str(key_cache->location));
1155 }
1156 
1157 static void
1158 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1159           const char *location)
1160 {
1161     key_cache->updated = time(NULL);
1162     key_cache->location = location;
1163     copy_gnutls_datum(&(key_cache->key), key);
1164     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1165               crm_str(location));
1166 }
1167 
1168 /*!
1169  * \internal
1170  * \brief Get Pacemaker Remote authentication key from file or cache
1171  *
1172  * \param[in]  location         Path to key file to try (this memory must
1173  *                              persist across all calls of this function)
1174  * \param[out] key              Key from location or cache
1175  *
1176  * \return Standard Pacemaker return code
1177  */
1178 static int
1179 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1180 {
1181     static struct key_cache_s key_cache = { 0, };
1182     int rc = pcmk_rc_ok;
1183 
1184     if ((location == NULL) || (key == NULL)) {
1185         return EINVAL;
1186     }
1187 
1188     if (key_is_cached(&key_cache)) {
1189         if (key_cache_expired(&key_cache)) {
1190             clear_key_cache(&key_cache);
1191         } else {
1192             get_cached_key(&key_cache, key);
1193             return pcmk_rc_ok;
1194         }
1195     }
1196 
1197     rc = read_gnutls_key(location, key);
1198     if (rc != pcmk_rc_ok) {
1199         return rc;
1200     }
1201     cache_key(&key_cache, key, location);
1202     return pcmk_rc_ok;
1203 }
1204 
1205 /*!
1206  * \internal
1207  * \brief Initialize the Pacemaker Remote authentication key
1208  *
1209  * Try loading the Pacemaker Remote authentication key from cache if available,
1210  * otherwise from these locations, in order of preference: the value of the
1211  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1212  * file location; or (for historical reasons) /etc/corosync/authkey.
1213  *
1214  * \param[out] key  Where to store key
1215  *
1216  * \return Standard Pacemaker return code
1217  */
1218 int
1219 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1220 {
1221     static const char *env_location = NULL;
1222     static bool need_env = true;
1223 
1224     int env_rc = pcmk_rc_ok;
1225     int default_rc = pcmk_rc_ok;
1226     int alt_rc = pcmk_rc_ok;
1227 
1228     bool env_is_default = false;
1229     bool env_is_fallback = false;
1230 
1231     if (need_env) {
1232         env_location = getenv("PCMK_authkey_location");
1233         need_env = false;
1234     }
1235 
1236     // Try location in environment variable, if set
1237     if (env_location != NULL) {
1238         env_rc = get_remote_key(env_location, key);
1239         if (env_rc == pcmk_rc_ok) {
1240             return pcmk_rc_ok;
1241         }
1242 
1243         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1244         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1245 
1246         /* @TODO It would be more secure to fail, rather than fall back to the
1247          * default, if an explicitly set key location is not readable, and it
1248          * would be better to never use the Corosync location as a fallback.
1249          * However, that would break any deployments currently working with the
1250          * fallbacks.
1251          */
1252     }
1253 
1254     // Try default location, if environment wasn't explicitly set to it
1255     if (env_is_default) {
1256         default_rc = env_rc;
1257     } else {
1258         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1259     }
1260 
1261     // Try fallback location, if environment wasn't set to it and default failed
1262     if (env_is_fallback) {
1263         alt_rc = env_rc;
1264     } else if (default_rc != pcmk_rc_ok) {
1265         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1266     }
1267 
1268     // We have all results, so log and return
1269 
1270     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1271         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1272 
1273         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1274                  env_location,
1275                  env_is_default? "" : "or default location ",
1276                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1277                  !env_is_default && !env_is_fallback? " " : "",
1278                  env_is_fallback? "" : "or fallback location ",
1279                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1280                  pcmk_rc_str(env_rc));
1281         return ENOKEY;
1282     }
1283 
1284     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1285         crm_warn("Could not read Pacemaker Remote key from %s "
1286                  "(using %s location %s instead): %s",
1287                  env_location,
1288                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1289                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1290                  pcmk_rc_str(env_rc));
1291         return pcmk_rc_ok;
1292     }
1293 
1294     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
1295         // Environment unset, defaults failed
1296         crm_warn("Could not read Pacemaker Remote key from default location %s"
1297                  " (or fallback location %s): %s",
1298                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1299                  pcmk_rc_str(default_rc));
1300         return ENOKEY;
1301     }
1302 
1303     return pcmk_rc_ok; // Environment variable unset, a default worked
1304 }
1305 
1306 static void
1307 lrmd_gnutls_global_init(void)
     /*  */
1308 {
1309     static int gnutls_init = 0;
1310 
1311     if (!gnutls_init) {
1312         crm_gnutls_global_init();
1313     }
1314     gnutls_init = 1;
1315 }
1316 #endif
1317 
1318 static void
1319 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1320 {
1321     lrmd_private_t *native = lrmd->lrmd_private;
1322 
1323     if (native->callback) {
1324         lrmd_event_data_t event = { 0, };
1325         event.type = lrmd_event_connect;
1326         event.remote_nodename = native->remote_nodename;
1327         event.connection_rc = rc;
1328         native->callback(&event);
1329     }
1330 }
1331 
1332 #ifdef HAVE_GNUTLS_GNUTLS_H
1333 static inline int
1334 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1335 {
1336     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1337 }
1338 
1339 /*!
1340  * \internal
1341  * \brief Add trigger and file descriptor mainloop sources for TLS
1342  *
1343  * \param[in] lrmd          API connection with established TLS session
1344  * \param[in] do_handshake  Whether to perform executor handshake
1345  *
1346  * \return Standard Pacemaker return code
1347  */
1348 static int
1349 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1350 {
1351     lrmd_private_t *native = lrmd->lrmd_private;
1352     int rc = pcmk_rc_ok;
1353 
1354     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1355                                    native->server, native->port);
1356 
1357     struct mainloop_fd_callbacks tls_fd_callbacks = {
1358         .dispatch = lrmd_tls_dispatch,
1359         .destroy = lrmd_tls_connection_destroy,
1360     };
1361 
1362     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1363                                                   lrmd_tls_dispatch, lrmd);
1364     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1365                                      &tls_fd_callbacks);
1366 
1367     /* Async connections lose the client name provided by the API caller, so we
1368      * have to use our generated name here to perform the executor handshake.
1369      *
1370      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1371      * that name in this function instead of generating one anyway.
1372      */
1373     if (do_handshake) {
1374         rc = lrmd_handshake(lrmd, name);
1375         rc = pcmk_legacy2rc(rc);
1376     }
1377     free(name);
1378     return rc;
1379 }
1380 
1381 static void
1382 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1383 {
1384     lrmd_t *lrmd = userdata;
1385     lrmd_private_t *native = lrmd->lrmd_private;
1386     gnutls_datum_t psk_key = { NULL, 0 };
1387 
1388     native->async_timer = 0;
1389 
1390     if (rc != pcmk_rc_ok) {
1391         lrmd_tls_connection_destroy(lrmd);
1392         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1393                  CRM_XS " rc=%d",
1394                  native->server, native->port, pcmk_rc_str(rc), rc);
1395         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1396         return;
1397     }
1398 
1399     /* The TCP connection was successful, so establish the TLS connection.
1400      * @TODO make this async to avoid blocking code in client
1401      */
1402 
1403     native->sock = sock;
1404 
1405     rc = lrmd__init_remote_key(&psk_key);
1406     if (rc != pcmk_rc_ok) {
1407         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1408                  CRM_XS " rc=%d",
1409                  native->server, native->port, pcmk_rc_str(rc), rc);
1410         lrmd_tls_connection_destroy(lrmd);
1411         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1412         return;
1413     }
1414 
1415     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1416     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1417     gnutls_free(psk_key.data);
1418 
1419     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1420                                                         GNUTLS_CRD_PSK,
1421                                                         native->psk_cred_c);
1422     if (native->remote->tls_session == NULL) {
1423         lrmd_tls_connection_destroy(lrmd);
1424         report_async_connection_result(lrmd, -EPROTO);
1425         return;
1426     }
1427 
1428     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1429         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1430                  native->server, native->port);
1431         gnutls_deinit(*native->remote->tls_session);
1432         gnutls_free(native->remote->tls_session);
1433         native->remote->tls_session = NULL;
1434         lrmd_tls_connection_destroy(lrmd);
1435         report_async_connection_result(lrmd, -EKEYREJECTED);
1436         return;
1437     }
1438 
1439     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1440              native->server, native->port);
1441     rc = add_tls_to_mainloop(lrmd, true);
1442     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1443 }
1444 
1445 static int
1446 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1447 {
1448     int rc;
1449     int timer_id = 0;
1450     lrmd_private_t *native = lrmd->lrmd_private;
1451 
1452     lrmd_gnutls_global_init();
1453     native->sock = -1;
1454     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1455                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1456     if (rc != pcmk_rc_ok) {
1457         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1458                  CRM_XS " rc=%d",
1459                  native->server, native->port, pcmk_rc_str(rc), rc);
1460         return pcmk_rc2legacy(rc);
1461     }
1462     native->async_timer = timer_id;
1463     return pcmk_ok;
1464 }
1465 
1466 static int
1467 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1468 {
1469     int rc;
1470 
1471     lrmd_private_t *native = lrmd->lrmd_private;
1472     gnutls_datum_t psk_key = { NULL, 0 };
1473 
1474     lrmd_gnutls_global_init();
1475 
1476     native->sock = -1;
1477     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1478                               &(native->sock), NULL, NULL);
1479     if (rc != pcmk_rc_ok) {
1480         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1481                  CRM_XS " rc=%d",
1482                  native->server, native->port, pcmk_rc_str(rc), rc);
1483         lrmd_tls_connection_destroy(lrmd);
1484         return -ENOTCONN;
1485     }
1486 
1487     rc = lrmd__init_remote_key(&psk_key);
1488     if (rc != pcmk_rc_ok) {
1489         lrmd_tls_connection_destroy(lrmd);
1490         return pcmk_rc2legacy(rc);
1491     }
1492 
1493     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1494     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1495     gnutls_free(psk_key.data);
1496 
1497     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1498                                                         GNUTLS_CRD_PSK,
1499                                                         native->psk_cred_c);
1500     if (native->remote->tls_session == NULL) {
1501         lrmd_tls_connection_destroy(lrmd);
1502         return -EPROTO;
1503     }
1504 
1505     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1506         crm_err("Session creation for %s:%d failed", native->server, native->port);
1507         gnutls_deinit(*native->remote->tls_session);
1508         gnutls_free(native->remote->tls_session);
1509         native->remote->tls_session = NULL;
1510         lrmd_tls_connection_destroy(lrmd);
1511         return -EKEYREJECTED;
1512     }
1513 
1514     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1515              native->port);
1516 
1517     if (fd) {
1518         *fd = native->sock;
1519     } else {
1520         add_tls_to_mainloop(lrmd, false);
1521     }
1522     return pcmk_ok;
1523 }
1524 #endif
1525 
1526 static int
1527 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1528 {
1529     int rc = -ENOTCONN;
1530     lrmd_private_t *native = lrmd->lrmd_private;
1531 
1532     switch (native->type) {
1533         case pcmk__client_ipc:
1534             rc = lrmd_ipc_connect(lrmd, fd);
1535             break;
1536 #ifdef HAVE_GNUTLS_GNUTLS_H
1537         case pcmk__client_tls:
1538             rc = lrmd_tls_connect(lrmd, fd);
1539             break;
1540 #endif
1541         default:
1542             crm_err("Unsupported executor connection type (bug?): %d",
1543                     native->type);
1544             rc = -EPROTONOSUPPORT;
1545     }
1546 
1547     if (rc == pcmk_ok) {
1548         rc = lrmd_handshake(lrmd, name);
1549     }
1550 
1551     return rc;
1552 }
1553 
1554 static int
1555 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1556 {
1557     int rc = pcmk_ok;
1558     lrmd_private_t *native = lrmd->lrmd_private;
1559 
1560     CRM_CHECK(native && native->callback, return -EINVAL);
1561 
1562     switch (native->type) {
1563         case pcmk__client_ipc:
1564             /* fake async connection with ipc.  it should be fast
1565              * enough that we gain very little from async */
1566             rc = lrmd_api_connect(lrmd, name, NULL);
1567             if (!rc) {
1568                 report_async_connection_result(lrmd, rc);
1569             }
1570             break;
1571 #ifdef HAVE_GNUTLS_GNUTLS_H
1572         case pcmk__client_tls:
1573             rc = lrmd_tls_connect_async(lrmd, timeout);
1574             if (rc) {
1575                 /* connection failed, report rc now */
1576                 report_async_connection_result(lrmd, rc);
1577             }
1578             break;
1579 #endif
1580         default:
1581             crm_err("Unsupported executor connection type (bug?): %d",
1582                     native->type);
1583             rc = -EPROTONOSUPPORT;
1584     }
1585 
1586     return rc;
1587 }
1588 
1589 static void
1590 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1591 {
1592     lrmd_private_t *native = lrmd->lrmd_private;
1593 
1594     if (native->source != NULL) {
1595         /* Attached to mainloop */
1596         mainloop_del_ipc_client(native->source);
1597         native->source = NULL;
1598         native->ipc = NULL;
1599 
1600     } else if (native->ipc) {
1601         /* Not attached to mainloop */
1602         crm_ipc_t *ipc = native->ipc;
1603 
1604         native->ipc = NULL;
1605         crm_ipc_close(ipc);
1606         crm_ipc_destroy(ipc);
1607     }
1608 }
1609 
1610 #ifdef HAVE_GNUTLS_GNUTLS_H
1611 static void
1612 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1613 {
1614     lrmd_private_t *native = lrmd->lrmd_private;
1615 
1616     if (native->remote->tls_session) {
1617         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1618         gnutls_deinit(*native->remote->tls_session);
1619         gnutls_free(native->remote->tls_session);
1620         native->remote->tls_session = 0;
1621     }
1622 
1623     if (native->async_timer) {
1624         g_source_remove(native->async_timer);
1625         native->async_timer = 0;
1626     }
1627 
1628     if (native->source != NULL) {
1629         /* Attached to mainloop */
1630         mainloop_del_ipc_client(native->source);
1631         native->source = NULL;
1632 
1633     } else if (native->sock) {
1634         close(native->sock);
1635         native->sock = 0;
1636     }
1637 
1638     if (native->pending_notify) {
1639         g_list_free_full(native->pending_notify, lrmd_free_xml);
1640         native->pending_notify = NULL;
1641     }
1642 }
1643 #endif
1644 
1645 static int
1646 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1647 {
1648     lrmd_private_t *native = lrmd->lrmd_private;
1649     int rc = pcmk_ok;
1650 
1651     crm_info("Disconnecting %s %s executor connection",
1652              pcmk__client_type_str(native->type),
1653              (native->remote_nodename? native->remote_nodename : "local"));
1654     switch (native->type) {
1655         case pcmk__client_ipc:
1656             lrmd_ipc_disconnect(lrmd);
1657             break;
1658 #ifdef HAVE_GNUTLS_GNUTLS_H
1659         case pcmk__client_tls:
1660             lrmd_tls_disconnect(lrmd);
1661             break;
1662 #endif
1663         default:
1664             crm_err("Unsupported executor connection type (bug?): %d",
1665                     native->type);
1666             rc = -EPROTONOSUPPORT;
1667     }
1668 
1669     free(native->token);
1670     native->token = NULL;
1671 
1672     free(native->peer_version);
1673     native->peer_version = NULL;
1674     return rc;
1675 }
1676 
1677 static int
1678 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1679                       const char *rsc_id,
1680                       const char *class,
1681                       const char *provider, const char *type, enum lrmd_call_options options)
1682 {
1683     int rc = pcmk_ok;
1684     xmlNode *data = NULL;
1685 
1686     if (!class || !type || !rsc_id) {
1687         return -EINVAL;
1688     }
1689     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1690         && (provider == NULL)) {
1691         return -EINVAL;
1692     }
1693 
1694     data = create_xml_node(NULL, F_LRMD_RSC);
1695 
1696     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1697     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1698     crm_xml_add(data, F_LRMD_CLASS, class);
1699     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1700     crm_xml_add(data, F_LRMD_TYPE, type);
1701     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1702     free_xml(data);
1703 
1704     return rc;
1705 }
1706 
1707 static int
1708 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1709 {
1710     int rc = pcmk_ok;
1711     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1712 
1713     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1714     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1715     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1716     free_xml(data);
1717 
1718     return rc;
1719 }
1720 
1721 lrmd_rsc_info_t *
1722 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1723                   const char *provider, const char *type)
1724 {
1725     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1726 
1727     CRM_ASSERT(rsc_info);
1728     pcmk__str_update(&rsc_info->id, rsc_id);
1729     pcmk__str_update(&rsc_info->standard, standard);
1730     pcmk__str_update(&rsc_info->provider, provider);
1731     pcmk__str_update(&rsc_info->type, type);
1732     return rsc_info;
1733 }
1734 
1735 lrmd_rsc_info_t *
1736 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1737 {
1738     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1739                              rsc_info->provider, rsc_info->type);
1740 }
1741 
1742 void
1743 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1744 {
1745     if (!rsc_info) {
1746         return;
1747     }
1748     free(rsc_info->id);
1749     free(rsc_info->type);
1750     free(rsc_info->standard);
1751     free(rsc_info->provider);
1752     free(rsc_info);
1753 }
1754 
1755 static lrmd_rsc_info_t *
1756 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1757 {
1758     lrmd_rsc_info_t *rsc_info = NULL;
1759     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1760     xmlNode *output = NULL;
1761     const char *class = NULL;
1762     const char *provider = NULL;
1763     const char *type = NULL;
1764 
1765     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1766     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1767     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1768     free_xml(data);
1769 
1770     if (!output) {
1771         return NULL;
1772     }
1773 
1774     class = crm_element_value(output, F_LRMD_CLASS);
1775     provider = crm_element_value(output, F_LRMD_PROVIDER);
1776     type = crm_element_value(output, F_LRMD_TYPE);
1777 
1778     if (!class || !type) {
1779         free_xml(output);
1780         return NULL;
1781     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1782                && !provider) {
1783         free_xml(output);
1784         return NULL;
1785     }
1786 
1787     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1788     free_xml(output);
1789     return rsc_info;
1790 }
1791 
1792 void
1793 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1794 {
1795     if (op_info) {
1796         free(op_info->rsc_id);
1797         free(op_info->action);
1798         free(op_info->interval_ms_s);
1799         free(op_info->timeout_ms_s);
1800         free(op_info);
1801     }
1802 }
1803 
1804 static int
1805 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1806                            enum lrmd_call_options options, GList **output)
1807 {
1808     xmlNode *data = NULL;
1809     xmlNode *output_xml = NULL;
1810     int rc = pcmk_ok;
1811 
1812     if (output == NULL) {
1813         return -EINVAL;
1814     }
1815     *output = NULL;
1816 
1817     // Send request
1818     if (rsc_id) {
1819         data = create_xml_node(NULL, F_LRMD_RSC);
1820         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1821         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1822     }
1823     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1824                            timeout_ms, options, TRUE);
1825     if (data) {
1826         free_xml(data);
1827     }
1828 
1829     // Process reply
1830     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1831         return rc;
1832     }
1833     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1834          (rsc_xml != NULL) && (rc == pcmk_ok);
1835          rsc_xml = crm_next_same_xml(rsc_xml)) {
1836 
1837         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1838         if (rsc_id == NULL) {
1839             crm_err("Could not parse recurring operation information from executor");
1840             continue;
1841         }
1842         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1843              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1844 
1845             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1846 
1847             if (op_info == NULL) {
1848                 rc = -ENOMEM;
1849                 break;
1850             }
1851             op_info->rsc_id = strdup(rsc_id);
1852             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1853             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1854                                                             F_LRMD_RSC_INTERVAL);
1855             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1856                                                            F_LRMD_TIMEOUT);
1857             *output = g_list_prepend(*output, op_info);
1858         }
1859     }
1860     free_xml(output_xml);
1861     return rc;
1862 }
1863 
1864 
1865 static void
1866 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1867 {
1868     lrmd_private_t *native = lrmd->lrmd_private;
1869 
1870     native->callback = callback;
1871 }
1872 
1873 void
1874 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1875 {
1876     lrmd_private_t *native = lrmd->lrmd_private;
1877 
1878     native->proxy_callback = callback;
1879     native->proxy_callback_userdata = userdata;
1880 }
1881 
1882 void
1883 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1884 {
1885     lrmd_private_t *native = lrmd->lrmd_private;
1886 
1887     if (native->proxy_callback) {
1888         crm_log_xml_trace(msg, "PROXY_INBOUND");
1889         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1890     }
1891 }
1892 
1893 int
1894 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1895 {
1896     if (lrmd == NULL) {
1897         return -ENOTCONN;
1898     }
1899     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1900 
1901     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1902     return lrmd_send_xml_no_reply(lrmd, msg);
1903 }
1904 
1905 static int
1906 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1907 {
1908     int rc = pcmk_ok;
1909     stonith_t *stonith_api = stonith_api_new();
1910 
1911     if (stonith_api == NULL) {
1912         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1913         return -ENOMEM;
1914     }
1915 
1916     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1917                                      provider, output, 0);
1918     if ((rc == pcmk_ok) && (*output == NULL)) {
1919         rc = -EIO;
1920     }
1921     stonith_api->cmds->free(stonith_api);
1922     return rc;
1923 }
1924 
1925 static int
1926 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1927                       const char *type, char **output,
1928                       enum lrmd_call_options options)
1929 {
1930     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1931                                            output, options, NULL);
1932 }
1933 
1934 static int
1935 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1936                              const char *provider, const char *type,
1937                              char **output, enum lrmd_call_options options,
1938                              lrmd_key_value_t *params)
1939 {
1940     svc_action_t *action = NULL;
1941     GHashTable *params_table = NULL;
1942 
1943     if (!standard || !type) {
1944         lrmd_key_value_freeall(params);
1945         return -EINVAL;
1946     }
1947 
1948     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1949         lrmd_key_value_freeall(params);
1950         return stonith_get_metadata(provider, type, output);
1951     }
1952 
1953     params_table = pcmk__strkey_table(free, free);
1954     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1955         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1956     }
1957     action = services__create_resource_action(type, standard, provider, type,
1958                                               CRMD_ACTION_METADATA, 0,
1959                                               CRMD_METADATA_CALL_TIMEOUT,
1960                                               params_table, 0);
1961     lrmd_key_value_freeall(params);
1962 
1963     if (action == NULL) {
1964         return -ENOMEM;
1965     }
1966     if (action->rc != PCMK_OCF_UNKNOWN) {
1967         services_action_free(action);
1968         return -EINVAL;
1969     }
1970 
1971     if (!services_action_sync(action)) {
1972         crm_err("Failed to retrieve meta-data for %s:%s:%s",
1973                 standard, provider, type);
1974         services_action_free(action);
1975         return -EIO;
1976     }
1977 
1978     if (!action->stdout_data) {
1979         crm_err("Failed to receive meta-data for %s:%s:%s",
1980                 standard, provider, type);
1981         services_action_free(action);
1982         return -EIO;
1983     }
1984 
1985     *output = strdup(action->stdout_data);
1986     services_action_free(action);
1987 
1988     return pcmk_ok;
1989 }
1990 
1991 static int
1992 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
1993               const char *userdata, guint interval_ms,
1994               int timeout,      /* ms */
1995               int start_delay,  /* ms */
1996               enum lrmd_call_options options, lrmd_key_value_t * params)
1997 {
1998     int rc = pcmk_ok;
1999     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2000     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2001     lrmd_key_value_t *tmp = NULL;
2002 
2003     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2004     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2005     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2006     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
2007     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2008     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2009     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
2010 
2011     for (tmp = params; tmp; tmp = tmp->next) {
2012         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2013     }
2014 
2015     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2016     free_xml(data);
2017 
2018     lrmd_key_value_freeall(params);
2019     return rc;
2020 }
2021 
2022 /* timeout is in ms */
2023 static int
2024 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2025                     int timeout, lrmd_key_value_t *params)
2026 {
2027     int rc = pcmk_ok;
2028     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
2029     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2030     lrmd_key_value_t *tmp = NULL;
2031 
2032     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2033     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
2034     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
2035     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2036 
2037     for (tmp = params; tmp; tmp = tmp->next) {
2038         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2039     }
2040 
2041     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2042                            lrmd_opt_notify_orig_only, TRUE);
2043     free_xml(data);
2044 
2045     lrmd_key_value_freeall(params);
2046     return rc;
2047 }
2048 
2049 static int
2050 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2051                 guint interval_ms)
2052 {
2053     int rc = pcmk_ok;
2054     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2055 
2056     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2057     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2058     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2059     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2060     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2061     free_xml(data);
2062     return rc;
2063 }
2064 
2065 static int
2066 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2067 {
2068     int rc = 0;
2069     stonith_t *stonith_api = stonith_api_new();
2070     stonith_key_value_t *stonith_resources = NULL;
2071     stonith_key_value_t *dIter = NULL;
2072 
2073     if (stonith_api == NULL) {
2074         crm_err("Could not list fence agents: API memory allocation failed");
2075         return -ENOMEM;
2076     }
2077     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2078                                    &stonith_resources, 0);
2079     stonith_api->cmds->free(stonith_api);
2080 
2081     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2082         rc++;
2083         if (resources) {
2084             *resources = lrmd_list_add(*resources, dIter->value);
2085         }
2086     }
2087 
2088     stonith_key_value_freeall(stonith_resources, 1, 0);
2089     return rc;
2090 }
2091 
2092 static int
2093 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2094                      const char *provider)
2095 {
2096     int rc = 0;
2097     int stonith_count = 0; // Initially, whether to include stonith devices
2098 
2099     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2100         stonith_count = 1;
2101 
2102     } else {
2103         GList *gIter = NULL;
2104         GList *agents = resources_list_agents(class, provider);
2105 
2106         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2107             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2108             rc++;
2109         }
2110         g_list_free_full(agents, free);
2111 
2112         if (!class) {
2113             stonith_count = 1;
2114         }
2115     }
2116 
2117     if (stonith_count) {
2118         // Now, if stonith devices are included, how many there are
2119         stonith_count = list_stonith_agents(resources);
2120         if (stonith_count > 0) {
2121             rc += stonith_count;
2122         }
2123     }
2124     if (rc == 0) {
2125         crm_notice("No agents found for class %s", class);
2126         rc = -EPROTONOSUPPORT;
2127     }
2128     return rc;
2129 }
2130 
2131 static bool
2132 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2133 {
2134     bool found = false;
2135     GList *agents = NULL;
2136     GList *gIter2 = NULL;
2137 
2138     agents = resources_list_agents(class, provider);
2139     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2140         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2141             found = true;
2142         }
2143     }
2144     g_list_free_full(agents, free);
2145     return found;
2146 }
2147 
2148 static int
2149 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2150 {
2151     int rc = pcmk_ok;
2152     char *provider = NULL;
2153     GList *ocf_providers = NULL;
2154     GList *gIter = NULL;
2155 
2156     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2157 
2158     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2159         provider = gIter->data;
2160         if (!agent || does_provider_have_agent(agent, provider,
2161                                                PCMK_RESOURCE_CLASS_OCF)) {
2162             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2163             rc++;
2164         }
2165     }
2166 
2167     g_list_free_full(ocf_providers, free);
2168     return rc;
2169 }
2170 
2171 static int
2172 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2173 {
2174     int rc = 0;
2175     GList *standards = NULL;
2176     GList *gIter = NULL;
2177 
2178     standards = resources_list_standards();
2179 
2180     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2181         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2182         rc++;
2183     }
2184 
2185     if (list_stonith_agents(NULL) > 0) {
2186         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2187         rc++;
2188     }
2189 
2190     g_list_free_full(standards, free);
2191     return rc;
2192 }
2193 
2194 /*!
2195  * \internal
2196  * \brief Create an executor API object
2197  *
2198  * \param[out] api       Will be set to newly created API object (it is the
2199  *                       caller's responsibility to free this value with
2200  *                       lrmd_api_delete() if this function succeeds)
2201  * \param[in]  nodename  If the object will be used for a remote connection,
2202  *                       the node name to use in cluster for remote executor
2203  * \param[in]  server    If the object will be used for a remote connection,
2204  *                       the resolvable host name to connect to
2205  * \param[in]  port      If the object will be used for a remote connection,
2206  *                       port number on \p server to connect to
2207  *
2208  * \return Standard Pacemaker return code
2209  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2210  *       value will be used for both. If the caller leaves both NULL, an API
2211  *       object will be created for a local executor connection.
2212  */
2213 int
2214 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2215 {
2216     lrmd_private_t *pvt = NULL;
2217 
2218     if (api == NULL) {
2219         return EINVAL;
2220     }
2221     *api = NULL;
2222 
2223     // Allocate all memory needed
2224 
2225     *api = calloc(1, sizeof(lrmd_t));
2226     if (*api == NULL) {
2227         return ENOMEM;
2228     }
2229 
2230     pvt = calloc(1, sizeof(lrmd_private_t));
2231     if (pvt == NULL) {
2232         lrmd_api_delete(*api);
2233         *api = NULL;
2234         return ENOMEM;
2235     }
2236     (*api)->lrmd_private = pvt;
2237 
2238     // @TODO Do we need to do this for local connections?
2239     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2240 
2241     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2242 
2243     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2244         lrmd_api_delete(*api);
2245         *api = NULL;
2246         return ENOMEM;
2247     }
2248 
2249     // Set methods
2250     (*api)->cmds->connect = lrmd_api_connect;
2251     (*api)->cmds->connect_async = lrmd_api_connect_async;
2252     (*api)->cmds->is_connected = lrmd_api_is_connected;
2253     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2254     (*api)->cmds->disconnect = lrmd_api_disconnect;
2255     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2256     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2257     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2258     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2259     (*api)->cmds->set_callback = lrmd_api_set_callback;
2260     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2261     (*api)->cmds->exec = lrmd_api_exec;
2262     (*api)->cmds->cancel = lrmd_api_cancel;
2263     (*api)->cmds->list_agents = lrmd_api_list_agents;
2264     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2265     (*api)->cmds->list_standards = lrmd_api_list_standards;
2266     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2267     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2268 
2269     if ((nodename == NULL) && (server == NULL)) {
2270         pvt->type = pcmk__client_ipc;
2271     } else {
2272 #ifdef HAVE_GNUTLS_GNUTLS_H
2273         if (nodename == NULL) {
2274             nodename = server;
2275         } else if (server == NULL) {
2276             server = nodename;
2277         }
2278         pvt->type = pcmk__client_tls;
2279         pvt->remote_nodename = strdup(nodename);
2280         pvt->server = strdup(server);
2281         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2282             lrmd_api_delete(*api);
2283             *api = NULL;
2284             return ENOMEM;
2285         }
2286         pvt->port = port;
2287         if (pvt->port == 0) {
2288             pvt->port = crm_default_remote_port();
2289         }
2290 #else
2291         crm_err("Cannot communicate with Pacemaker Remote "
2292                 "because GnuTLS is not enabled for this build");
2293         lrmd_api_delete(*api);
2294         *api = NULL;
2295         return EOPNOTSUPP;
2296 #endif
2297     }
2298     return pcmk_rc_ok;
2299 }
2300 
2301 lrmd_t *
2302 lrmd_api_new(void)
     /*  */
2303 {
2304     lrmd_t *api = NULL;
2305 
2306     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2307     return api;
2308 }
2309 
2310 lrmd_t *
2311 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2312 {
2313     lrmd_t *api = NULL;
2314 
2315     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2316     return api;
2317 }
2318 
2319 void
2320 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2321 {
2322     if (lrmd == NULL) {
2323         return;
2324     }
2325     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2326         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2327             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2328         }
2329         free(lrmd->cmds);
2330     }
2331     if (lrmd->lrmd_private != NULL) {
2332         lrmd_private_t *native = lrmd->lrmd_private;
2333 
2334 #ifdef HAVE_GNUTLS_GNUTLS_H
2335         free(native->server);
2336 #endif
2337         free(native->remote_nodename);
2338         free(native->remote);
2339         free(native->token);
2340         free(native->peer_version);
2341         free(lrmd->lrmd_private);
2342     }
2343     free(lrmd);
2344 }
2345 
2346 /*!
2347  * \internal
2348  * \brief Set the result of an executor event
2349  *
2350  * \param[in,out] event        Executor event to set
2351  * \param[in]     rc           OCF exit status of event
2352  * \param[in]     op_status    Executor status of event
2353  * \param[in]     exit_reason  Human-friendly description of event
2354  */
2355 void
2356 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2357                  const char *exit_reason)
2358 {
2359     if (event == NULL) {
2360         return;
2361     }
2362 
2363     event->rc = rc;
2364     event->op_status = op_status;
2365     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2366 }
2367 
2368 /*!
2369  * \internal
2370  * \brief Clear an executor event's exit reason, output, and error output
2371  *
2372  * \param[in] event  Executor event to reset
2373  */
2374 void
2375 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2376 {
2377     if (event == NULL) {
2378         return;
2379     }
2380 
2381     free((void *) event->exit_reason);
2382     event->exit_reason = NULL;
2383 
2384     free((void *) event->output);
2385     event->output = NULL;
2386 }