root/lib/lrmd/lrmd_client.c

/* */

DEFINITIONS

1. lrmd_list_add
2. lrmd_list_freeall
3. lrmd_key_value_add
4. lrmd_key_value_freeall
5. lrmd_new_event
6. lrmd_copy_event
7. lrmd_free_event
8. lrmd_dispatch_internal
9. lrmd_ipc_dispatch
10. lrmd_free_xml
11. remote_executor_connected
12. lrmd_tls_dispatch
13. lrmd_poll
14. lrmd_dispatch
15. lrmd_create_op
16. lrmd_ipc_connection_destroy
17. lrmd_tls_connection_destroy
18. lrmd__remote_send_xml
19. read_remote_reply
20. send_remote_message
21. lrmd_tls_send_recv
22. lrmd_send_xml
23. lrmd_send_xml_no_reply
24. lrmd_api_is_connected
25. lrmd_send_command
26. lrmd_api_poke_connection
27. lrmd__validate_remote_settings
28. lrmd_handshake
29. lrmd_ipc_connect
30. copy_gnutls_datum
31. clear_gnutls_datum
32. read_gnutls_key
33. key_is_cached
34. key_cache_expired
35. clear_key_cache
36. get_cached_key
37. cache_key
38. get_remote_key
39. lrmd__init_remote_key
40. lrmd_gnutls_global_init
41. report_async_connection_result
42. lrmd__tls_client_handshake
43. add_tls_to_mainloop
44. lrmd_tcp_connect_cb
45. lrmd_tls_connect_async
46. lrmd_tls_connect
47. lrmd_api_connect
48. lrmd_api_connect_async
49. lrmd_ipc_disconnect
50. lrmd_tls_disconnect
51. lrmd_api_disconnect
52. lrmd_api_register_rsc
53. lrmd_api_unregister_rsc
54. lrmd_new_rsc_info
55. lrmd_copy_rsc_info
56. lrmd_free_rsc_info
57. lrmd_api_get_rsc_info
58. lrmd_free_op_info
59. lrmd_api_get_recurring_ops
60. lrmd_api_set_callback
61. lrmd_internal_set_proxy_callback
62. lrmd_internal_proxy_dispatch
63. lrmd_internal_proxy_send
64. stonith_get_metadata
65. lrmd_api_get_metadata
66. lrmd_api_get_metadata_params
67. lrmd_api_exec
68. lrmd_api_exec_alert
69. lrmd_api_cancel
70. list_stonith_agents
71. lrmd_api_list_agents
72. does_provider_have_agent
73. lrmd_api_list_ocf_providers
74. lrmd_api_list_standards
75. lrmd__new
76. lrmd_api_new
77. lrmd_remote_api_new
78. lrmd_api_delete
79. lrmd__set_result
80. lrmd__reset_result

   1 /*
   2  * Copyright 2012-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/msg_xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  undef KEYFILE
  42 #  include <gnutls/gnutls.h>
  43 #endif
  44 
  45 #include <sys/socket.h>
  46 #include <netinet/in.h>
  47 #include <netinet/ip.h>
  48 #include <arpa/inet.h>
  49 #include <netdb.h>
  50 
  51 #define MAX_TLS_RECV_WAIT 10000
  52 
  53 CRM_TRACE_INIT_DATA(lrmd);
  54 
  55 static int lrmd_api_disconnect(lrmd_t * lrmd);
  56 static int lrmd_api_is_connected(lrmd_t * lrmd);
  57 
  58 /* IPC proxy functions */
  59 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  60 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  61 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  62 
  63 #ifdef HAVE_GNUTLS_GNUTLS_H
  64 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  65 gnutls_psk_client_credentials_t psk_cred_s;
  66 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  67 static int global_remote_msg_id = 0;
  68 static void lrmd_tls_connection_destroy(gpointer userdata);
  69 #endif
  70 
  71 typedef struct lrmd_private_s {
  72     uint64_t type;
  73     char *token;
  74     mainloop_io_t *source;
  75 
  76     /* IPC parameters */
  77     crm_ipc_t *ipc;
  78 
  79     pcmk__remote_t *remote;
  80 
  81     /* Extra TLS parameters */
  82     char *remote_nodename;
  83 #ifdef HAVE_GNUTLS_GNUTLS_H
  84     char *server;
  85     int port;
  86     gnutls_psk_client_credentials_t psk_cred_c;
  87 
  88     /* while the async connection is occurring, this is the id
  89      * of the connection timeout timer. */
  90     int async_timer;
  91     int sock;
  92     /* since tls requires a round trip across the network for a
  93      * request/reply, there are times where we just want to be able
  94      * to send a request from the client and not wait around (or even care
  95      * about) what the reply is. */
  96     int expected_late_replies;
  97     GList *pending_notify;
  98     crm_trigger_t *process_notify;
  99 #endif
 100 
 101     lrmd_event_callback callback;
 102 
 103     /* Internal IPC proxy msg passing for remote guests */
 104     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 105     void *proxy_callback_userdata;
 106     char *peer_version;
 107 } lrmd_private_t;
 108 
 109 static lrmd_list_t *
 110 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 111 {
 112     lrmd_list_t *p, *end;
 113 
 114     p = calloc(1, sizeof(lrmd_list_t));
 115     p->val = strdup(value);
 116 
 117     end = head;
 118     while (end && end->next) {
 119         end = end->next;
 120     }
 121 
 122     if (end) {
 123         end->next = p;
 124     } else {
 125         head = p;
 126     }
 127 
 128     return head;
 129 }
 130 
 131 void
 132 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 133 {
 134     lrmd_list_t *p;
 135 
 136     while (head) {
 137         char *val = (char *)head->val;
 138 
 139         p = head->next;
 140         free(val);
 141         free(head);
 142         head = p;
 143     }
 144 }
 145 
 146 lrmd_key_value_t *
 147 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 148 {
 149     lrmd_key_value_t *p, *end;
 150 
 151     p = calloc(1, sizeof(lrmd_key_value_t));
 152     p->key = strdup(key);
 153     p->value = strdup(value);
 154 
 155     end = head;
 156     while (end && end->next) {
 157         end = end->next;
 158     }
 159 
 160     if (end) {
 161         end->next = p;
 162     } else {
 163         head = p;
 164     }
 165 
 166     return head;
 167 }
 168 
 169 void
 170 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 171 {
 172     lrmd_key_value_t *p;
 173 
 174     while (head) {
 175         p = head->next;
 176         free(head->key);
 177         free(head->value);
 178         free(head);
 179         head = p;
 180     }
 181 }
 182 
 183 /*!
 184  * Create a new lrmd_event_data_t object
 185  *
 186  * \param[in] rsc_id       ID of resource involved in event
 187  * \param[in] task         Action name
 188  * \param[in] interval_ms  Action interval
 189  *
 190  * \return Newly allocated and initialized lrmd_event_data_t
 191  * \note This functions asserts on memory errors, so the return value is
 192  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 193  *       result with lrmd_free_event().
 194  */
 195 lrmd_event_data_t *
 196 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 197 {
 198     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 199 
 200     CRM_ASSERT(event != NULL);
 201     if (rsc_id != NULL) {
 202         event->rsc_id = strdup(rsc_id);
 203         CRM_ASSERT(event->rsc_id != NULL);
 204     }
 205     if (task != NULL) {
 206         event->op_type = strdup(task);
 207         CRM_ASSERT(event->op_type != NULL);
 208     }
 209     event->interval_ms = interval_ms;
 210     return event;
 211 }
 212 
 213 lrmd_event_data_t *
 214 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 215 {
 216     lrmd_event_data_t *copy = NULL;
 217 
 218     copy = calloc(1, sizeof(lrmd_event_data_t));
 219 
 220     /* This will get all the int values.
 221      * we just have to be careful not to leave any
 222      * dangling pointers to strings. */
 223     memcpy(copy, event, sizeof(lrmd_event_data_t));
 224 
 225     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
 226     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
 227     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
 228     copy->output = event->output ? strdup(event->output) : NULL;
 229     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
 230     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
 231     copy->params = pcmk__str_table_dup(event->params);
 232 
 233     return copy;
 234 }
 235 
 236 /*!
 237  * \brief Free an executor event
 238  *
 239  * \param[in]  Executor event object to free
 240  */
 241 void
 242 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 243 {
 244     if (event == NULL) {
 245         return;
 246     }
 247     // @TODO Why are these const char *?
 248     free((void *) event->rsc_id);
 249     free((void *) event->op_type);
 250     free((void *) event->user_data);
 251     free((void *) event->remote_nodename);
 252     lrmd__reset_result(event);
 253     if (event->params != NULL) {
 254         g_hash_table_destroy(event->params);
 255     }
 256     free(event);
 257 }
 258 
 259 static void
 260 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 261 {
 262     const char *type;
 263     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 264     lrmd_private_t *native = lrmd->lrmd_private;
 265     lrmd_event_data_t event = { 0, };
 266 
 267     if (proxy_session != NULL) {
 268         /* this is proxy business */
 269         lrmd_internal_proxy_dispatch(lrmd, msg);
 270         return;
 271     } else if (!native->callback) {
 272         /* no callback set */
 273         crm_trace("notify event received but client has not set callback");
 274         return;
 275     }
 276 
 277     event.remote_nodename = native->remote_nodename;
 278     type = crm_element_value(msg, F_LRMD_OPERATION);
 279     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 280     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 281 
 282     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 283         event.type = lrmd_event_register;
 284     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 285         event.type = lrmd_event_unregister;
 286     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 287         time_t epoch = 0;
 288 
 289         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 290         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 291         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 292         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 293         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 294         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 295 
 296         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 297         event.t_run = (unsigned int) epoch;
 298 
 299         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 300         event.t_rcchange = (unsigned int) epoch;
 301 
 302         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 303         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 304 
 305         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 306         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 307         event.type = lrmd_event_exec_complete;
 308 
 309         // No need to duplicate the memory, so don't use setter functions
 310         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
 311         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
 312 
 313         event.params = xml2list(msg);
 314     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 315         event.type = lrmd_event_new_client;
 316     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 317         event.type = lrmd_event_poke;
 318     } else {
 319         return;
 320     }
 321 
 322     crm_trace("op %s notify event received", type);
 323     native->callback(&event);
 324 
 325     if (event.params) {
 326         g_hash_table_destroy(event.params);
 327     }
 328 }
 329 
 330 // \return Always 0, to indicate that IPC mainloop source should be kept
 331 static int
 332 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 333 {
 334     lrmd_t *lrmd = userdata;
 335     lrmd_private_t *native = lrmd->lrmd_private;
 336 
 337     if (native->callback != NULL) {
 338         xmlNode *msg = string2xml(buffer);
 339 
 340         lrmd_dispatch_internal(lrmd, msg);
 341         free_xml(msg);
 342     }
 343     return 0;
 344 }
 345 
 346 #ifdef HAVE_GNUTLS_GNUTLS_H
 347 static void
 348 lrmd_free_xml(gpointer userdata)
     /*  */
 349 {
 350     free_xml((xmlNode *) userdata);
 351 }
 352 
 353 static bool
 354 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 355 {
 356     lrmd_private_t *native = lrmd->lrmd_private;
 357 
 358     return (native->remote->tls_session != NULL);
 359 }
 360 
 361 /*!
 362  * \internal
 363  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 364  *
 365  * \param[in] userdata  API connection
 366  *
 367  * \return Always return a nonnegative value, which as a file descriptor
 368  *         dispatch function means keep the mainloop source, and as a
 369  *         trigger dispatch function, 0 means remove the trigger from the
 370  *         mainloop while 1 means keep it (and job completed)
 371  */
 372 static int
 373 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 374 {
 375     lrmd_t *lrmd = userdata;
 376     lrmd_private_t *native = lrmd->lrmd_private;
 377     xmlNode *xml = NULL;
 378     int rc = pcmk_rc_ok;
 379 
 380     if (!remote_executor_connected(lrmd)) {
 381         crm_trace("TLS dispatch triggered after disconnect");
 382         return 0;
 383     }
 384 
 385     crm_trace("TLS dispatch triggered");
 386 
 387     /* First check if there are any pending notifies to process that came
 388      * while we were waiting for replies earlier. */
 389     if (native->pending_notify) {
 390         GList *iter = NULL;
 391 
 392         crm_trace("Processing pending notifies");
 393         for (iter = native->pending_notify; iter; iter = iter->next) {
 394             lrmd_dispatch_internal(lrmd, iter->data);
 395         }
 396         g_list_free_full(native->pending_notify, lrmd_free_xml);
 397         native->pending_notify = NULL;
 398     }
 399 
 400     /* Next read the current buffer and see if there are any messages to handle. */
 401     switch (pcmk__remote_ready(native->remote, 0)) {
 402         case pcmk_rc_ok:
 403             rc = pcmk__read_remote_message(native->remote, -1);
 404             xml = pcmk__remote_message_xml(native->remote);
 405             break;
 406         case ETIME:
 407             // Nothing to read, check if a full message is already in buffer
 408             xml = pcmk__remote_message_xml(native->remote);
 409             break;
 410         default:
 411             rc = ENOTCONN;
 412             break;
 413     }
 414     while (xml) {
 415         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 416         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 417             lrmd_dispatch_internal(lrmd, xml);
 418         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 419             if (native->expected_late_replies > 0) {
 420                 native->expected_late_replies--;
 421             } else {
 422                 int reply_id = 0;
 423                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 424                 /* if this happens, we want to know about it */
 425                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 426             }
 427         }
 428         free_xml(xml);
 429         xml = pcmk__remote_message_xml(native->remote);
 430     }
 431 
 432     if (rc == ENOTCONN) {
 433         crm_info("Lost %s executor connection while reading data",
 434                  (native->remote_nodename? native->remote_nodename : "local"));
 435         lrmd_tls_disconnect(lrmd);
 436         return 0;
 437     }
 438     return 1;
 439 }
 440 #endif
 441 
 442 /* Not used with mainloop */
 443 int
 444 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 445 {
 446     lrmd_private_t *native = lrmd->lrmd_private;
 447 
 448     switch (native->type) {
 449         case pcmk__client_ipc:
 450             return crm_ipc_ready(native->ipc);
 451 
 452 #ifdef HAVE_GNUTLS_GNUTLS_H
 453         case pcmk__client_tls:
 454             if (native->pending_notify) {
 455                 return 1;
 456             } else {
 457                 int rc = pcmk__remote_ready(native->remote, 0);
 458 
 459                 switch (rc) {
 460                     case pcmk_rc_ok:
 461                         return 1;
 462                     case ETIME:
 463                         return 0;
 464                     default:
 465                         return pcmk_rc2legacy(rc);
 466                 }
 467             }
 468 #endif
 469         default:
 470             crm_err("Unsupported executor connection type (bug?): %d",
 471                     native->type);
 472             return -EPROTONOSUPPORT;
 473     }
 474 }
 475 
 476 /* Not used with mainloop */
 477 bool
 478 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 479 {
 480     lrmd_private_t *private = NULL;
 481 
 482     CRM_ASSERT(lrmd != NULL);
 483 
 484     private = lrmd->lrmd_private;
 485     switch (private->type) {
 486         case pcmk__client_ipc:
 487             while (crm_ipc_ready(private->ipc)) {
 488                 if (crm_ipc_read(private->ipc) > 0) {
 489                     const char *msg = crm_ipc_buffer(private->ipc);
 490 
 491                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 492                 }
 493             }
 494             break;
 495 #ifdef HAVE_GNUTLS_GNUTLS_H
 496         case pcmk__client_tls:
 497             lrmd_tls_dispatch(lrmd);
 498             break;
 499 #endif
 500         default:
 501             crm_err("Unsupported executor connection type (bug?): %d",
 502                     private->type);
 503     }
 504 
 505     if (lrmd_api_is_connected(lrmd) == FALSE) {
 506         crm_err("Connection closed");
 507         return FALSE;
 508     }
 509 
 510     return TRUE;
 511 }
 512 
 513 static xmlNode *
 514 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 515                enum lrmd_call_options options)
 516 {
 517     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 518 
 519     CRM_CHECK(op_msg != NULL, return NULL);
 520     CRM_CHECK(token != NULL, return NULL);
 521 
 522     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 523     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 524     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 525     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 526     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 527     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 528 
 529     if (data != NULL) {
 530         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 531     }
 532 
 533     crm_trace("Created executor %s command with call options %.8lx (%d)",
 534               op, (long)options, options);
 535     return op_msg;
 536 }
 537 
 538 static void
 539 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 540 {
 541     lrmd_t *lrmd = userdata;
 542     lrmd_private_t *native = lrmd->lrmd_private;
 543 
 544     crm_info("IPC connection destroyed");
 545 
 546     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 547     native->ipc = NULL;
 548     native->source = NULL;
 549 
 550     if (native->callback) {
 551         lrmd_event_data_t event = { 0, };
 552         event.type = lrmd_event_disconnect;
 553         event.remote_nodename = native->remote_nodename;
 554         native->callback(&event);
 555     }
 556 }
 557 
 558 #ifdef HAVE_GNUTLS_GNUTLS_H
 559 static void
 560 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 561 {
 562     lrmd_t *lrmd = userdata;
 563     lrmd_private_t *native = lrmd->lrmd_private;
 564 
 565     crm_info("TLS connection destroyed");
 566 
 567     if (native->remote->tls_session) {
 568         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 569         gnutls_deinit(*native->remote->tls_session);
 570         gnutls_free(native->remote->tls_session);
 571     }
 572     if (native->psk_cred_c) {
 573         gnutls_psk_free_client_credentials(native->psk_cred_c);
 574     }
 575     if (native->sock) {
 576         close(native->sock);
 577     }
 578     if (native->process_notify) {
 579         mainloop_destroy_trigger(native->process_notify);
 580         native->process_notify = NULL;
 581     }
 582     if (native->pending_notify) {
 583         g_list_free_full(native->pending_notify, lrmd_free_xml);
 584         native->pending_notify = NULL;
 585     }
 586 
 587     free(native->remote->buffer);
 588     native->remote->buffer = NULL;
 589     native->source = 0;
 590     native->sock = 0;
 591     native->psk_cred_c = NULL;
 592     native->remote->tls_session = NULL;
 593     native->sock = 0;
 594 
 595     if (native->callback) {
 596         lrmd_event_data_t event = { 0, };
 597         event.remote_nodename = native->remote_nodename;
 598         event.type = lrmd_event_disconnect;
 599         native->callback(&event);
 600     }
 601     return;
 602 }
 603 
 604 // \return Standard Pacemaker return code
 605 int
 606 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 607                       const char *msg_type)
 608 {
 609     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 610     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 611     return pcmk__remote_send_xml(session, msg);
 612 }
 613 
 614 // \return Standard Pacemaker return code
 615 static int
 616 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 617                   xmlNode **reply)
 618 {
 619     lrmd_private_t *native = lrmd->lrmd_private;
 620     time_t start = time(NULL);
 621     const char *msg_type = NULL;
 622     int reply_id = 0;
 623     int remaining_timeout = 0;
 624     int rc = pcmk_rc_ok;
 625 
 626     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 627      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 628     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 629         total_timeout = MAX_TLS_RECV_WAIT;
 630     }
 631 
 632     for (*reply = NULL; *reply == NULL; ) {
 633 
 634         *reply = pcmk__remote_message_xml(native->remote);
 635         if (*reply == NULL) {
 636             /* read some more off the tls buffer if we still have time left. */
 637             if (remaining_timeout) {
 638                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 639             } else {
 640                 remaining_timeout = total_timeout;
 641             }
 642             if (remaining_timeout <= 0) {
 643                 return ETIME;
 644             }
 645 
 646             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 647             if (rc != pcmk_rc_ok) {
 648                 return rc;
 649             }
 650 
 651             *reply = pcmk__remote_message_xml(native->remote);
 652             if (*reply == NULL) {
 653                 return ENOMSG;
 654             }
 655         }
 656 
 657         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
 658         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 659 
 660         if (!msg_type) {
 661             crm_err("Empty msg type received while waiting for reply");
 662             free_xml(*reply);
 663             *reply = NULL;
 664         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 665             /* got a notify while waiting for reply, trigger the notify to be processed later */
 666             crm_info("queueing notify");
 667             native->pending_notify = g_list_append(native->pending_notify, *reply);
 668             if (native->process_notify) {
 669                 crm_info("notify trigger set.");
 670                 mainloop_set_trigger(native->process_notify);
 671             }
 672             *reply = NULL;
 673         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 674             /* msg isn't a reply, make some noise */
 675             crm_err("Expected a reply, got %s", msg_type);
 676             free_xml(*reply);
 677             *reply = NULL;
 678         } else if (reply_id != expected_reply_id) {
 679             if (native->expected_late_replies > 0) {
 680                 native->expected_late_replies--;
 681             } else {
 682                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 683             }
 684             free_xml(*reply);
 685             *reply = NULL;
 686         }
 687     }
 688 
 689     if (native->remote->buffer && native->process_notify) {
 690         mainloop_set_trigger(native->process_notify);
 691     }
 692 
 693     return rc;
 694 }
 695 
 696 // \return Standard Pacemaker return code
 697 static int
 698 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 699 {
 700     int rc = pcmk_rc_ok;
 701     lrmd_private_t *native = lrmd->lrmd_private;
 702 
 703     global_remote_msg_id++;
 704     if (global_remote_msg_id <= 0) {
 705         global_remote_msg_id = 1;
 706     }
 707 
 708     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 709                                "request");
 710     if (rc != pcmk_rc_ok) {
 711         crm_err("Disconnecting because TLS message could not be sent to "
 712                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 713         lrmd_tls_disconnect(lrmd);
 714     }
 715     return rc;
 716 }
 717 
 718 static int
 719 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 720 {
 721     int rc = 0;
 722     xmlNode *xml = NULL;
 723 
 724     if (!remote_executor_connected(lrmd)) {
 725         return -ENOTCONN;
 726     }
 727 
 728     rc = send_remote_message(lrmd, msg);
 729     if (rc != pcmk_rc_ok) {
 730         return pcmk_rc2legacy(rc);
 731     }
 732 
 733     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 734     if (rc != pcmk_rc_ok) {
 735         crm_err("Disconnecting remote after request %d reply not received: %s "
 736                 CRM_XS " rc=%d timeout=%dms",
 737                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 738         lrmd_tls_disconnect(lrmd);
 739     }
 740 
 741     if (reply) {
 742         *reply = xml;
 743     } else {
 744         free_xml(xml);
 745     }
 746 
 747     return pcmk_rc2legacy(rc);
 748 }
 749 #endif
 750 
 751 static int
 752 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 753 {
 754     int rc = pcmk_ok;
 755     lrmd_private_t *native = lrmd->lrmd_private;
 756 
 757     switch (native->type) {
 758         case pcmk__client_ipc:
 759             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 760             break;
 761 #ifdef HAVE_GNUTLS_GNUTLS_H
 762         case pcmk__client_tls:
 763             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 764             break;
 765 #endif
 766         default:
 767             crm_err("Unsupported executor connection type (bug?): %d",
 768                     native->type);
 769             rc = -EPROTONOSUPPORT;
 770     }
 771 
 772     return rc;
 773 }
 774 
 775 static int
 776 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 777 {
 778     int rc = pcmk_ok;
 779     lrmd_private_t *native = lrmd->lrmd_private;
 780 
 781     switch (native->type) {
 782         case pcmk__client_ipc:
 783             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 784             break;
 785 #ifdef HAVE_GNUTLS_GNUTLS_H
 786         case pcmk__client_tls:
 787             rc = send_remote_message(lrmd, msg);
 788             if (rc == pcmk_rc_ok) {
 789                 /* we don't want to wait around for the reply, but
 790                  * since the request/reply protocol needs to behave the same
 791                  * as libqb, a reply will eventually come later anyway. */
 792                 native->expected_late_replies++;
 793             }
 794             rc = pcmk_rc2legacy(rc);
 795             break;
 796 #endif
 797         default:
 798             crm_err("Unsupported executor connection type (bug?): %d",
 799                     native->type);
 800             rc = -EPROTONOSUPPORT;
 801     }
 802 
 803     return rc;
 804 }
 805 
 806 static int
 807 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 808 {
 809     lrmd_private_t *native = lrmd->lrmd_private;
 810 
 811     switch (native->type) {
 812         case pcmk__client_ipc:
 813             return crm_ipc_connected(native->ipc);
 814 #ifdef HAVE_GNUTLS_GNUTLS_H
 815         case pcmk__client_tls:
 816             return remote_executor_connected(lrmd);
 817 #endif
 818         default:
 819             crm_err("Unsupported executor connection type (bug?): %d",
 820                     native->type);
 821             return 0;
 822     }
 823 }
 824 
 825 /*!
 826  * \internal
 827  * \brief Send a prepared API command to the executor
 828  *
 829  * \param[in]  lrmd          Existing connection to the executor
 830  * \param[in]  op            Name of API command to send
 831  * \param[in]  data          Command data XML to add to the sent command
 832  * \param[out] output_data   If expecting a reply, it will be stored here
 833  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to
 834  *                           a sensible value per the type of connection,
 835  *                           standard vs. pacemaker remote);
 836  *                           also propagated to the command XML
 837  * \param[in]  call_options  Call options to pass to server when sending
 838  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
 839  *                           must be TRUE for IPC (as opposed to TLS) clients
 840  *
 841  * \return pcmk_ok on success, -errno on error
 842  */
 843 static int
 844 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 845                   xmlNode **output_data, int timeout,
 846                   enum lrmd_call_options options, gboolean expect_reply)
 847 {
 848     int rc = pcmk_ok;
 849     lrmd_private_t *native = lrmd->lrmd_private;
 850     xmlNode *op_msg = NULL;
 851     xmlNode *op_reply = NULL;
 852 
 853     if (!lrmd_api_is_connected(lrmd)) {
 854         return -ENOTCONN;
 855     }
 856 
 857     if (op == NULL) {
 858         crm_err("No operation specified");
 859         return -EINVAL;
 860     }
 861 
 862     CRM_CHECK(native->token != NULL,;
 863         );
 864     crm_trace("Sending %s op to executor", op);
 865 
 866     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 867 
 868     if (op_msg == NULL) {
 869         return -EINVAL;
 870     }
 871 
 872     if (expect_reply) {
 873         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 874     } else {
 875         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 876         goto done;
 877     }
 878 
 879     if (rc < 0) {
 880         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 881         goto done;
 882 
 883     } else if(op_reply == NULL) {
 884         rc = -ENOMSG;
 885         goto done;
 886     }
 887 
 888     rc = pcmk_ok;
 889     crm_trace("%s op reply received", op);
 890     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 891         rc = -ENOMSG;
 892         goto done;
 893     }
 894 
 895     crm_log_xml_trace(op_reply, "Reply");
 896 
 897     if (output_data) {
 898         *output_data = op_reply;
 899         op_reply = NULL;        /* Prevent subsequent free */
 900     }
 901 
 902   done:
 903     if (lrmd_api_is_connected(lrmd) == FALSE) {
 904         crm_err("Executor disconnected");
 905     }
 906 
 907     free_xml(op_msg);
 908     free_xml(op_reply);
 909     return rc;
 910 }
 911 
 912 static int
 913 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 914 {
 915     int rc;
 916     lrmd_private_t *native = lrmd->lrmd_private;
 917     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 918 
 919     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 920     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 921                            (native->type == pcmk__client_ipc));
 922     free_xml(data);
 923 
 924     return rc < 0 ? rc : pcmk_ok;
 925 }
 926 
 927 // \return Standard Pacemaker return code
 928 int
 929 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 930 {
 931     int rc = pcmk_rc_ok;
 932     const char *value;
 933     lrmd_private_t *native = lrmd->lrmd_private;
 934     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 935 
 936     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 937 
 938     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 939     if ((value) &&
 940         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
 941        crm_xml_add(data, F_LRMD_WATCHDOG, value);
 942     }
 943 
 944     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 945                            (native->type == pcmk__client_ipc));
 946     free_xml(data);
 947     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 948 }
 949 
 950 static int
 951 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 952 {
 953     int rc = pcmk_ok;
 954     lrmd_private_t *native = lrmd->lrmd_private;
 955     xmlNode *reply = NULL;
 956     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 957 
 958     crm_xml_add(hello, F_TYPE, T_LRMD);
 959     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 960     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 961     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 962 
 963     /* advertise that we are a proxy provider */
 964     if (native->proxy_callback) {
 965         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
 966     }
 967 
 968     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 969 
 970     if (rc < 0) {
 971         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 972         rc = -ECOMM;
 973     } else if (reply == NULL) {
 974         crm_err("Did not receive registration reply");
 975         rc = -EPROTO;
 976     } else {
 977         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 978         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 979         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 980 
 981         crm_element_value_int(reply, F_LRMD_RC, &rc);
 982 
 983         if (rc == -EPROTO) {
 984             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
 985                 LRMD_PROTOCOL_VERSION, version);
 986             crm_log_xml_err(reply, "Protocol Error");
 987 
 988         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
 989             crm_err("Invalid registration message: %s", msg_type);
 990             crm_log_xml_err(reply, "Bad reply");
 991             rc = -EPROTO;
 992         } else if (tmp_ticket == NULL) {
 993             crm_err("No registration token provided");
 994             crm_log_xml_err(reply, "Bad reply");
 995             rc = -EPROTO;
 996         } else {
 997             crm_trace("Obtained registration token: %s", tmp_ticket);
 998             native->token = strdup(tmp_ticket);
 999             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1000             rc = pcmk_ok;
1001         }
1002     }
1003 
1004     free_xml(reply);
1005     free_xml(hello);
1006 
1007     if (rc != pcmk_ok) {
1008         lrmd_api_disconnect(lrmd);
1009     }
1010     return rc;
1011 }
1012 
1013 static int
1014 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1015 {
1016     int rc = pcmk_ok;
1017     lrmd_private_t *native = lrmd->lrmd_private;
1018 
1019     struct ipc_client_callbacks lrmd_callbacks = {
1020         .dispatch = lrmd_ipc_dispatch,
1021         .destroy = lrmd_ipc_connection_destroy
1022     };
1023 
1024     crm_info("Connecting to executor");
1025 
1026     if (fd) {
1027         /* No mainloop */
1028         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1029         if (native->ipc && crm_ipc_connect(native->ipc)) {
1030             *fd = crm_ipc_get_fd(native->ipc);
1031         } else if (native->ipc) {
1032             crm_perror(LOG_ERR, "Connection to executor failed");
1033             rc = -ENOTCONN;
1034         }
1035     } else {
1036         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1037         native->ipc = mainloop_get_ipc_client(native->source);
1038     }
1039 
1040     if (native->ipc == NULL) {
1041         crm_debug("Could not connect to the executor API");
1042         rc = -ENOTCONN;
1043     }
1044 
1045     return rc;
1046 }
1047 
1048 #ifdef HAVE_GNUTLS_GNUTLS_H
1049 static void
1050 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1051 {
1052     CRM_ASSERT((dest != NULL) && (source != NULL) && (source->data != NULL));
1053 
1054     dest->data = gnutls_malloc(source->size);
1055     CRM_ASSERT(dest->data);
1056 
1057     memcpy(dest->data, source->data, source->size);
1058     dest->size = source->size;
1059 }
1060 
1061 static void
1062 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1063 {
1064     gnutls_free(datum->data);
1065     datum->data = NULL;
1066     datum->size = 0;
1067 }
1068 
1069 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1070 
1071 // \return Standard Pacemaker return code
1072 static int
1073 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1074 {
1075     FILE *stream = NULL;
1076     size_t buf_len = KEY_READ_LEN;
1077 
1078     if ((location == NULL) || (key == NULL)) {
1079         return EINVAL;
1080     }
1081 
1082     stream = fopen(location, "r");
1083     if (stream == NULL) {
1084         return errno;
1085     }
1086 
1087     key->data = gnutls_malloc(buf_len);
1088     key->size = 0;
1089     while (!feof(stream)) {
1090         int next = fgetc(stream);
1091 
1092         if (next == EOF) {
1093             if (!feof(stream)) {
1094                 crm_warn("Pacemaker Remote key read was partially successful "
1095                          "(copy in memory may be corrupted)");
1096             }
1097             break;
1098         }
1099         if (key->size == buf_len) {
1100             buf_len = key->size + KEY_READ_LEN;
1101             key->data = gnutls_realloc(key->data, buf_len);
1102             CRM_ASSERT(key->data);
1103         }
1104         key->data[key->size++] = (unsigned char) next;
1105     }
1106     fclose(stream);
1107 
1108     if (key->size == 0) {
1109         clear_gnutls_datum(key);
1110         return ENOKEY;
1111     }
1112     return pcmk_rc_ok;
1113 }
1114 
1115 // Cache the most recently used Pacemaker Remote authentication key
1116 
1117 struct key_cache_s {
1118     time_t updated;         // When cached key was read (valid for 1 minute)
1119     const char *location;   // Where cached key was read from
1120     gnutls_datum_t key;     // Cached key
1121 };
1122 
1123 static bool
1124 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1125 {
1126     return key_cache->updated != 0;
1127 }
1128 
1129 static bool
1130 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1131 {
1132     return (time(NULL) - key_cache->updated) >= 60;
1133 }
1134 
1135 static void
1136 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1137 {
1138     clear_gnutls_datum(&(key_cache->key));
1139     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1140         key_cache->updated = 0;
1141         key_cache->location = NULL;
1142         crm_debug("Cleared Pacemaker Remote key cache");
1143     }
1144 }
1145 
1146 static void
1147 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1148 {
1149     copy_gnutls_datum(key, &(key_cache->key));
1150     crm_debug("Using cached Pacemaker Remote key from %s",
1151               crm_str(key_cache->location));
1152 }
1153 
1154 static void
1155 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1156           const char *location)
1157 {
1158     key_cache->updated = time(NULL);
1159     key_cache->location = location;
1160     copy_gnutls_datum(&(key_cache->key), key);
1161     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1162               crm_str(location));
1163 }
1164 
1165 /*!
1166  * \internal
1167  * \brief Get Pacemaker Remote authentication key from file or cache
1168  *
1169  * \param[in]  location         Path to key file to try (this memory must
1170  *                              persist across all calls of this function)
1171  * \param[out] key              Key from location or cache
1172  *
1173  * \return Standard Pacemaker return code
1174  */
1175 static int
1176 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1177 {
1178     static struct key_cache_s key_cache = { 0, };
1179     int rc = pcmk_rc_ok;
1180 
1181     if ((location == NULL) || (key == NULL)) {
1182         return EINVAL;
1183     }
1184 
1185     if (key_is_cached(&key_cache)) {
1186         if (key_cache_expired(&key_cache)) {
1187             clear_key_cache(&key_cache);
1188         } else {
1189             get_cached_key(&key_cache, key);
1190             return pcmk_rc_ok;
1191         }
1192     }
1193 
1194     rc = read_gnutls_key(location, key);
1195     if (rc != pcmk_rc_ok) {
1196         return rc;
1197     }
1198     cache_key(&key_cache, key, location);
1199     return pcmk_rc_ok;
1200 }
1201 
1202 /*!
1203  * \internal
1204  * \brief Initialize the Pacemaker Remote authentication key
1205  *
1206  * Try loading the Pacemaker Remote authentication key from cache if available,
1207  * otherwise from these locations, in order of preference: the value of the
1208  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1209  * file location; or (for historical reasons) /etc/corosync/authkey.
1210  *
1211  * \param[out] key  Where to store key
1212  *
1213  * \return Standard Pacemaker return code
1214  */
1215 int
1216 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1217 {
1218     static const char *env_location = NULL;
1219     static bool need_env = true;
1220 
1221     int env_rc = pcmk_rc_ok;
1222     int default_rc = pcmk_rc_ok;
1223     int alt_rc = pcmk_rc_ok;
1224 
1225     bool env_is_default = false;
1226     bool env_is_fallback = false;
1227 
1228     if (need_env) {
1229         env_location = getenv("PCMK_authkey_location");
1230         need_env = false;
1231     }
1232 
1233     // Try location in environment variable, if set
1234     if (env_location != NULL) {
1235         env_rc = get_remote_key(env_location, key);
1236         if (env_rc == pcmk_rc_ok) {
1237             return pcmk_rc_ok;
1238         }
1239 
1240         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1241         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1242 
1243         /* @TODO It would be more secure to fail, rather than fall back to the
1244          * default, if an explicitly set key location is not readable, and it
1245          * would be better to never use the Corosync location as a fallback.
1246          * However, that would break any deployments currently working with the
1247          * fallbacks.
1248          */
1249     }
1250 
1251     // Try default location, if environment wasn't explicitly set to it
1252     if (env_is_default) {
1253         default_rc = env_rc;
1254     } else {
1255         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1256     }
1257 
1258     // Try fallback location, if environment wasn't set to it and default failed
1259     if (env_is_fallback) {
1260         alt_rc = env_rc;
1261     } else if (default_rc != pcmk_rc_ok) {
1262         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1263     }
1264 
1265     // We have all results, so log and return
1266 
1267     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1268         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1269 
1270         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1271                  env_location,
1272                  env_is_default? "" : "or default location ",
1273                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1274                  !env_is_default && !env_is_fallback? " " : "",
1275                  env_is_fallback? "" : "or fallback location ",
1276                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1277                  pcmk_rc_str(env_rc));
1278         return ENOKEY;
1279     }
1280 
1281     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1282         crm_warn("Could not read Pacemaker Remote key from %s "
1283                  "(using %s location %s instead): %s",
1284                  env_location,
1285                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1286                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1287                  pcmk_rc_str(env_rc));
1288         return pcmk_rc_ok;
1289     }
1290 
1291     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
1292         // Environment unset, defaults failed
1293         crm_warn("Could not read Pacemaker Remote key from default location %s"
1294                  " (or fallback location %s): %s",
1295                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1296                  pcmk_rc_str(default_rc));
1297         return ENOKEY;
1298     }
1299 
1300     return pcmk_rc_ok; // Environment variable unset, a default worked
1301 }
1302 
1303 static void
1304 lrmd_gnutls_global_init(void)
     /*  */
1305 {
1306     static int gnutls_init = 0;
1307 
1308     if (!gnutls_init) {
1309         crm_gnutls_global_init();
1310     }
1311     gnutls_init = 1;
1312 }
1313 #endif
1314 
1315 static void
1316 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1317 {
1318     lrmd_private_t *native = lrmd->lrmd_private;
1319 
1320     if (native->callback) {
1321         lrmd_event_data_t event = { 0, };
1322         event.type = lrmd_event_connect;
1323         event.remote_nodename = native->remote_nodename;
1324         event.connection_rc = rc;
1325         native->callback(&event);
1326     }
1327 }
1328 
1329 #ifdef HAVE_GNUTLS_GNUTLS_H
1330 static inline int
1331 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1332 {
1333     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1334 }
1335 
1336 /*!
1337  * \internal
1338  * \brief Add trigger and file descriptor mainloop sources for TLS
1339  *
1340  * \param[in] lrmd          API connection with established TLS session
1341  * \param[in] do_handshake  Whether to perform executor handshake
1342  *
1343  * \return Standard Pacemaker return code
1344  */
1345 static int
1346 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1347 {
1348     lrmd_private_t *native = lrmd->lrmd_private;
1349     int rc = pcmk_rc_ok;
1350 
1351     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1352                                    native->server, native->port);
1353 
1354     struct mainloop_fd_callbacks tls_fd_callbacks = {
1355         .dispatch = lrmd_tls_dispatch,
1356         .destroy = lrmd_tls_connection_destroy,
1357     };
1358 
1359     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1360                                                   lrmd_tls_dispatch, lrmd);
1361     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1362                                      &tls_fd_callbacks);
1363 
1364     /* Async connections lose the client name provided by the API caller, so we
1365      * have to use our generated name here to perform the executor handshake.
1366      *
1367      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1368      * that name in this function instead of generating one anyway.
1369      */
1370     if (do_handshake) {
1371         rc = lrmd_handshake(lrmd, name);
1372         rc = pcmk_legacy2rc(rc);
1373     }
1374     free(name);
1375     return rc;
1376 }
1377 
1378 static void
1379 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1380 {
1381     lrmd_t *lrmd = userdata;
1382     lrmd_private_t *native = lrmd->lrmd_private;
1383     gnutls_datum_t psk_key = { NULL, 0 };
1384 
1385     native->async_timer = 0;
1386 
1387     if (rc != pcmk_rc_ok) {
1388         lrmd_tls_connection_destroy(lrmd);
1389         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1390                  CRM_XS " rc=%d",
1391                  native->server, native->port, pcmk_rc_str(rc), rc);
1392         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1393         return;
1394     }
1395 
1396     /* The TCP connection was successful, so establish the TLS connection.
1397      * @TODO make this async to avoid blocking code in client
1398      */
1399 
1400     native->sock = sock;
1401 
1402     rc = lrmd__init_remote_key(&psk_key);
1403     if (rc != pcmk_rc_ok) {
1404         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1405                  CRM_XS " rc=%d",
1406                  native->server, native->port, pcmk_rc_str(rc), rc);
1407         lrmd_tls_connection_destroy(lrmd);
1408         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1409         return;
1410     }
1411 
1412     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1413     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1414     gnutls_free(psk_key.data);
1415 
1416     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1417                                                         GNUTLS_CRD_PSK,
1418                                                         native->psk_cred_c);
1419     if (native->remote->tls_session == NULL) {
1420         lrmd_tls_connection_destroy(lrmd);
1421         report_async_connection_result(lrmd, -EPROTO);
1422         return;
1423     }
1424 
1425     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1426         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1427                  native->server, native->port);
1428         gnutls_deinit(*native->remote->tls_session);
1429         gnutls_free(native->remote->tls_session);
1430         native->remote->tls_session = NULL;
1431         lrmd_tls_connection_destroy(lrmd);
1432         report_async_connection_result(lrmd, -EKEYREJECTED);
1433         return;
1434     }
1435 
1436     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1437              native->server, native->port);
1438     rc = add_tls_to_mainloop(lrmd, true);
1439     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1440 }
1441 
1442 static int
1443 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1444 {
1445     int rc;
1446     int timer_id = 0;
1447     lrmd_private_t *native = lrmd->lrmd_private;
1448 
1449     lrmd_gnutls_global_init();
1450     native->sock = -1;
1451     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1452                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1453     if (rc != pcmk_rc_ok) {
1454         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1455                  CRM_XS " rc=%d",
1456                  native->server, native->port, pcmk_rc_str(rc), rc);
1457         return pcmk_rc2legacy(rc);
1458     }
1459     native->async_timer = timer_id;
1460     return pcmk_ok;
1461 }
1462 
1463 static int
1464 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1465 {
1466     int rc;
1467 
1468     lrmd_private_t *native = lrmd->lrmd_private;
1469     gnutls_datum_t psk_key = { NULL, 0 };
1470 
1471     lrmd_gnutls_global_init();
1472 
1473     native->sock = -1;
1474     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1475                               &(native->sock), NULL, NULL);
1476     if (rc != pcmk_rc_ok) {
1477         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1478                  CRM_XS " rc=%d",
1479                  native->server, native->port, pcmk_rc_str(rc), rc);
1480         lrmd_tls_connection_destroy(lrmd);
1481         return -ENOTCONN;
1482     }
1483 
1484     rc = lrmd__init_remote_key(&psk_key);
1485     if (rc != pcmk_rc_ok) {
1486         lrmd_tls_connection_destroy(lrmd);
1487         return pcmk_rc2legacy(rc);
1488     }
1489 
1490     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1491     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1492     gnutls_free(psk_key.data);
1493 
1494     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1495                                                         GNUTLS_CRD_PSK,
1496                                                         native->psk_cred_c);
1497     if (native->remote->tls_session == NULL) {
1498         lrmd_tls_connection_destroy(lrmd);
1499         return -EPROTO;
1500     }
1501 
1502     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1503         crm_err("Session creation for %s:%d failed", native->server, native->port);
1504         gnutls_deinit(*native->remote->tls_session);
1505         gnutls_free(native->remote->tls_session);
1506         native->remote->tls_session = NULL;
1507         lrmd_tls_connection_destroy(lrmd);
1508         return -EKEYREJECTED;
1509     }
1510 
1511     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1512              native->port);
1513 
1514     if (fd) {
1515         *fd = native->sock;
1516     } else {
1517         add_tls_to_mainloop(lrmd, false);
1518     }
1519     return pcmk_ok;
1520 }
1521 #endif
1522 
1523 static int
1524 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1525 {
1526     int rc = -ENOTCONN;
1527     lrmd_private_t *native = lrmd->lrmd_private;
1528 
1529     switch (native->type) {
1530         case pcmk__client_ipc:
1531             rc = lrmd_ipc_connect(lrmd, fd);
1532             break;
1533 #ifdef HAVE_GNUTLS_GNUTLS_H
1534         case pcmk__client_tls:
1535             rc = lrmd_tls_connect(lrmd, fd);
1536             break;
1537 #endif
1538         default:
1539             crm_err("Unsupported executor connection type (bug?): %d",
1540                     native->type);
1541             rc = -EPROTONOSUPPORT;
1542     }
1543 
1544     if (rc == pcmk_ok) {
1545         rc = lrmd_handshake(lrmd, name);
1546     }
1547 
1548     return rc;
1549 }
1550 
1551 static int
1552 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1553 {
1554     int rc = pcmk_ok;
1555     lrmd_private_t *native = lrmd->lrmd_private;
1556 
1557     CRM_CHECK(native && native->callback, return -EINVAL);
1558 
1559     switch (native->type) {
1560         case pcmk__client_ipc:
1561             /* fake async connection with ipc.  it should be fast
1562              * enough that we gain very little from async */
1563             rc = lrmd_api_connect(lrmd, name, NULL);
1564             if (!rc) {
1565                 report_async_connection_result(lrmd, rc);
1566             }
1567             break;
1568 #ifdef HAVE_GNUTLS_GNUTLS_H
1569         case pcmk__client_tls:
1570             rc = lrmd_tls_connect_async(lrmd, timeout);
1571             if (rc) {
1572                 /* connection failed, report rc now */
1573                 report_async_connection_result(lrmd, rc);
1574             }
1575             break;
1576 #endif
1577         default:
1578             crm_err("Unsupported executor connection type (bug?): %d",
1579                     native->type);
1580             rc = -EPROTONOSUPPORT;
1581     }
1582 
1583     return rc;
1584 }
1585 
1586 static void
1587 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1588 {
1589     lrmd_private_t *native = lrmd->lrmd_private;
1590 
1591     if (native->source != NULL) {
1592         /* Attached to mainloop */
1593         mainloop_del_ipc_client(native->source);
1594         native->source = NULL;
1595         native->ipc = NULL;
1596 
1597     } else if (native->ipc) {
1598         /* Not attached to mainloop */
1599         crm_ipc_t *ipc = native->ipc;
1600 
1601         native->ipc = NULL;
1602         crm_ipc_close(ipc);
1603         crm_ipc_destroy(ipc);
1604     }
1605 }
1606 
1607 #ifdef HAVE_GNUTLS_GNUTLS_H
1608 static void
1609 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1610 {
1611     lrmd_private_t *native = lrmd->lrmd_private;
1612 
1613     if (native->remote->tls_session) {
1614         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1615         gnutls_deinit(*native->remote->tls_session);
1616         gnutls_free(native->remote->tls_session);
1617         native->remote->tls_session = 0;
1618     }
1619 
1620     if (native->async_timer) {
1621         g_source_remove(native->async_timer);
1622         native->async_timer = 0;
1623     }
1624 
1625     if (native->source != NULL) {
1626         /* Attached to mainloop */
1627         mainloop_del_ipc_client(native->source);
1628         native->source = NULL;
1629 
1630     } else if (native->sock) {
1631         close(native->sock);
1632         native->sock = 0;
1633     }
1634 
1635     if (native->pending_notify) {
1636         g_list_free_full(native->pending_notify, lrmd_free_xml);
1637         native->pending_notify = NULL;
1638     }
1639 }
1640 #endif
1641 
1642 static int
1643 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1644 {
1645     lrmd_private_t *native = lrmd->lrmd_private;
1646     int rc = pcmk_ok;
1647 
1648     crm_info("Disconnecting %s %s executor connection",
1649              pcmk__client_type_str(native->type),
1650              (native->remote_nodename? native->remote_nodename : "local"));
1651     switch (native->type) {
1652         case pcmk__client_ipc:
1653             lrmd_ipc_disconnect(lrmd);
1654             break;
1655 #ifdef HAVE_GNUTLS_GNUTLS_H
1656         case pcmk__client_tls:
1657             lrmd_tls_disconnect(lrmd);
1658             break;
1659 #endif
1660         default:
1661             crm_err("Unsupported executor connection type (bug?): %d",
1662                     native->type);
1663             rc = -EPROTONOSUPPORT;
1664     }
1665 
1666     free(native->token);
1667     native->token = NULL;
1668 
1669     free(native->peer_version);
1670     native->peer_version = NULL;
1671     return rc;
1672 }
1673 
1674 static int
1675 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1676                       const char *rsc_id,
1677                       const char *class,
1678                       const char *provider, const char *type, enum lrmd_call_options options)
1679 {
1680     int rc = pcmk_ok;
1681     xmlNode *data = NULL;
1682 
1683     if (!class || !type || !rsc_id) {
1684         return -EINVAL;
1685     }
1686     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1687         && (provider == NULL)) {
1688         return -EINVAL;
1689     }
1690 
1691     data = create_xml_node(NULL, F_LRMD_RSC);
1692 
1693     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1694     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1695     crm_xml_add(data, F_LRMD_CLASS, class);
1696     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1697     crm_xml_add(data, F_LRMD_TYPE, type);
1698     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1699     free_xml(data);
1700 
1701     return rc;
1702 }
1703 
1704 static int
1705 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1706 {
1707     int rc = pcmk_ok;
1708     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1709 
1710     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1711     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1712     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1713     free_xml(data);
1714 
1715     return rc;
1716 }
1717 
1718 lrmd_rsc_info_t *
1719 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1720                   const char *provider, const char *type)
1721 {
1722     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1723 
1724     CRM_ASSERT(rsc_info);
1725     if (rsc_id) {
1726         rsc_info->id = strdup(rsc_id);
1727         CRM_ASSERT(rsc_info->id);
1728     }
1729     if (standard) {
1730         rsc_info->standard = strdup(standard);
1731         CRM_ASSERT(rsc_info->standard);
1732     }
1733     if (provider) {
1734         rsc_info->provider = strdup(provider);
1735         CRM_ASSERT(rsc_info->provider);
1736     }
1737     if (type) {
1738         rsc_info->type = strdup(type);
1739         CRM_ASSERT(rsc_info->type);
1740     }
1741     return rsc_info;
1742 }
1743 
1744 lrmd_rsc_info_t *
1745 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1746 {
1747     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1748                              rsc_info->provider, rsc_info->type);
1749 }
1750 
1751 void
1752 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1753 {
1754     if (!rsc_info) {
1755         return;
1756     }
1757     free(rsc_info->id);
1758     free(rsc_info->type);
1759     free(rsc_info->standard);
1760     free(rsc_info->provider);
1761     free(rsc_info);
1762 }
1763 
1764 static lrmd_rsc_info_t *
1765 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1766 {
1767     lrmd_rsc_info_t *rsc_info = NULL;
1768     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1769     xmlNode *output = NULL;
1770     const char *class = NULL;
1771     const char *provider = NULL;
1772     const char *type = NULL;
1773 
1774     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1775     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1776     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1777     free_xml(data);
1778 
1779     if (!output) {
1780         return NULL;
1781     }
1782 
1783     class = crm_element_value(output, F_LRMD_CLASS);
1784     provider = crm_element_value(output, F_LRMD_PROVIDER);
1785     type = crm_element_value(output, F_LRMD_TYPE);
1786 
1787     if (!class || !type) {
1788         free_xml(output);
1789         return NULL;
1790     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1791                && !provider) {
1792         free_xml(output);
1793         return NULL;
1794     }
1795 
1796     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1797     free_xml(output);
1798     return rsc_info;
1799 }
1800 
1801 void
1802 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1803 {
1804     if (op_info) {
1805         free(op_info->rsc_id);
1806         free(op_info->action);
1807         free(op_info->interval_ms_s);
1808         free(op_info->timeout_ms_s);
1809         free(op_info);
1810     }
1811 }
1812 
1813 static int
1814 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1815                            enum lrmd_call_options options, GList **output)
1816 {
1817     xmlNode *data = NULL;
1818     xmlNode *output_xml = NULL;
1819     int rc = pcmk_ok;
1820 
1821     if (output == NULL) {
1822         return -EINVAL;
1823     }
1824     *output = NULL;
1825 
1826     // Send request
1827     if (rsc_id) {
1828         data = create_xml_node(NULL, F_LRMD_RSC);
1829         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1830         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1831     }
1832     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1833                            timeout_ms, options, TRUE);
1834     if (data) {
1835         free_xml(data);
1836     }
1837 
1838     // Process reply
1839     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1840         return rc;
1841     }
1842     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1843          (rsc_xml != NULL) && (rc == pcmk_ok);
1844          rsc_xml = crm_next_same_xml(rsc_xml)) {
1845 
1846         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1847         if (rsc_id == NULL) {
1848             crm_err("Could not parse recurring operation information from executor");
1849             continue;
1850         }
1851         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1852              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1853 
1854             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1855 
1856             if (op_info == NULL) {
1857                 rc = -ENOMEM;
1858                 break;
1859             }
1860             op_info->rsc_id = strdup(rsc_id);
1861             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1862             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1863                                                             F_LRMD_RSC_INTERVAL);
1864             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1865                                                            F_LRMD_TIMEOUT);
1866             *output = g_list_prepend(*output, op_info);
1867         }
1868     }
1869     free_xml(output_xml);
1870     return rc;
1871 }
1872 
1873 
1874 static void
1875 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1876 {
1877     lrmd_private_t *native = lrmd->lrmd_private;
1878 
1879     native->callback = callback;
1880 }
1881 
1882 void
1883 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1884 {
1885     lrmd_private_t *native = lrmd->lrmd_private;
1886 
1887     native->proxy_callback = callback;
1888     native->proxy_callback_userdata = userdata;
1889 }
1890 
1891 void
1892 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1893 {
1894     lrmd_private_t *native = lrmd->lrmd_private;
1895 
1896     if (native->proxy_callback) {
1897         crm_log_xml_trace(msg, "PROXY_INBOUND");
1898         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1899     }
1900 }
1901 
1902 int
1903 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1904 {
1905     if (lrmd == NULL) {
1906         return -ENOTCONN;
1907     }
1908     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1909 
1910     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1911     return lrmd_send_xml_no_reply(lrmd, msg);
1912 }
1913 
1914 static int
1915 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1916 {
1917     int rc = pcmk_ok;
1918     stonith_t *stonith_api = stonith_api_new();
1919 
1920     if (stonith_api == NULL) {
1921         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1922         return -ENOMEM;
1923     }
1924 
1925     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1926                                      provider, output, 0);
1927     if ((rc == pcmk_ok) && (*output == NULL)) {
1928         rc = -EIO;
1929     }
1930     stonith_api->cmds->free(stonith_api);
1931     return rc;
1932 }
1933 
1934 static int
1935 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1936                       const char *type, char **output,
1937                       enum lrmd_call_options options)
1938 {
1939     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1940                                            output, options, NULL);
1941 }
1942 
1943 static int
1944 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1945                              const char *provider, const char *type,
1946                              char **output, enum lrmd_call_options options,
1947                              lrmd_key_value_t *params)
1948 {
1949     svc_action_t *action = NULL;
1950     GHashTable *params_table = NULL;
1951 
1952     if (!standard || !type) {
1953         lrmd_key_value_freeall(params);
1954         return -EINVAL;
1955     }
1956 
1957     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1958         lrmd_key_value_freeall(params);
1959         return stonith_get_metadata(provider, type, output);
1960     }
1961 
1962     params_table = pcmk__strkey_table(free, free);
1963     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1964         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1965     }
1966     action = services__create_resource_action(type, standard, provider, type,
1967                                               CRMD_ACTION_METADATA, 0,
1968                                               CRMD_METADATA_CALL_TIMEOUT,
1969                                               params_table, 0);
1970     lrmd_key_value_freeall(params);
1971 
1972     if (action == NULL) {
1973         return -ENOMEM;
1974     }
1975     if (action->rc != PCMK_OCF_UNKNOWN) {
1976         services_action_free(action);
1977         return -EINVAL;
1978     }
1979 
1980     if (!services_action_sync(action)) {
1981         crm_err("Failed to retrieve meta-data for %s:%s:%s",
1982                 standard, provider, type);
1983         services_action_free(action);
1984         return -EIO;
1985     }
1986 
1987     if (!action->stdout_data) {
1988         crm_err("Failed to receive meta-data for %s:%s:%s",
1989                 standard, provider, type);
1990         services_action_free(action);
1991         return -EIO;
1992     }
1993 
1994     *output = strdup(action->stdout_data);
1995     services_action_free(action);
1996 
1997     return pcmk_ok;
1998 }
1999 
2000 static int
2001 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2002               const char *userdata, guint interval_ms,
2003               int timeout,      /* ms */
2004               int start_delay,  /* ms */
2005               enum lrmd_call_options options, lrmd_key_value_t * params)
2006 {
2007     int rc = pcmk_ok;
2008     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2009     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2010     lrmd_key_value_t *tmp = NULL;
2011 
2012     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2013     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2014     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2015     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
2016     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2017     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2018     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
2019 
2020     for (tmp = params; tmp; tmp = tmp->next) {
2021         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2022     }
2023 
2024     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2025     free_xml(data);
2026 
2027     lrmd_key_value_freeall(params);
2028     return rc;
2029 }
2030 
2031 /* timeout is in ms */
2032 static int
2033 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2034                     int timeout, lrmd_key_value_t *params)
2035 {
2036     int rc = pcmk_ok;
2037     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
2038     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2039     lrmd_key_value_t *tmp = NULL;
2040 
2041     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2042     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
2043     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
2044     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2045 
2046     for (tmp = params; tmp; tmp = tmp->next) {
2047         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2048     }
2049 
2050     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2051                            lrmd_opt_notify_orig_only, TRUE);
2052     free_xml(data);
2053 
2054     lrmd_key_value_freeall(params);
2055     return rc;
2056 }
2057 
2058 static int
2059 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2060                 guint interval_ms)
2061 {
2062     int rc = pcmk_ok;
2063     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2064 
2065     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2066     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2067     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2068     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2069     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2070     free_xml(data);
2071     return rc;
2072 }
2073 
2074 static int
2075 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2076 {
2077     int rc = 0;
2078     stonith_t *stonith_api = stonith_api_new();
2079     stonith_key_value_t *stonith_resources = NULL;
2080     stonith_key_value_t *dIter = NULL;
2081 
2082     if (stonith_api == NULL) {
2083         crm_err("Could not list fence agents: API memory allocation failed");
2084         return -ENOMEM;
2085     }
2086     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2087                                    &stonith_resources, 0);
2088     stonith_api->cmds->free(stonith_api);
2089 
2090     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2091         rc++;
2092         if (resources) {
2093             *resources = lrmd_list_add(*resources, dIter->value);
2094         }
2095     }
2096 
2097     stonith_key_value_freeall(stonith_resources, 1, 0);
2098     return rc;
2099 }
2100 
2101 static int
2102 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2103                      const char *provider)
2104 {
2105     int rc = 0;
2106     int stonith_count = 0; // Initially, whether to include stonith devices
2107 
2108     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2109         stonith_count = 1;
2110 
2111     } else {
2112         GList *gIter = NULL;
2113         GList *agents = resources_list_agents(class, provider);
2114 
2115         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2116             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2117             rc++;
2118         }
2119         g_list_free_full(agents, free);
2120 
2121         if (!class) {
2122             stonith_count = 1;
2123         }
2124     }
2125 
2126     if (stonith_count) {
2127         // Now, if stonith devices are included, how many there are
2128         stonith_count = list_stonith_agents(resources);
2129         if (stonith_count > 0) {
2130             rc += stonith_count;
2131         }
2132     }
2133     if (rc == 0) {
2134         crm_notice("No agents found for class %s", class);
2135         rc = -EPROTONOSUPPORT;
2136     }
2137     return rc;
2138 }
2139 
2140 static bool
2141 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2142 {
2143     bool found = false;
2144     GList *agents = NULL;
2145     GList *gIter2 = NULL;
2146 
2147     agents = resources_list_agents(class, provider);
2148     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2149         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2150             found = true;
2151         }
2152     }
2153     g_list_free_full(agents, free);
2154     return found;
2155 }
2156 
2157 static int
2158 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2159 {
2160     int rc = pcmk_ok;
2161     char *provider = NULL;
2162     GList *ocf_providers = NULL;
2163     GList *gIter = NULL;
2164 
2165     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2166 
2167     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2168         provider = gIter->data;
2169         if (!agent || does_provider_have_agent(agent, provider,
2170                                                PCMK_RESOURCE_CLASS_OCF)) {
2171             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2172             rc++;
2173         }
2174     }
2175 
2176     g_list_free_full(ocf_providers, free);
2177     return rc;
2178 }
2179 
2180 static int
2181 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2182 {
2183     int rc = 0;
2184     GList *standards = NULL;
2185     GList *gIter = NULL;
2186 
2187     standards = resources_list_standards();
2188 
2189     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2190         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2191         rc++;
2192     }
2193 
2194     if (list_stonith_agents(NULL) > 0) {
2195         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2196         rc++;
2197     }
2198 
2199     g_list_free_full(standards, free);
2200     return rc;
2201 }
2202 
2203 /*!
2204  * \internal
2205  * \brief Create an executor API object
2206  *
2207  * \param[out] api       Will be set to newly created API object (it is the
2208  *                       caller's responsibility to free this value with
2209  *                       lrmd_api_delete() if this function succeeds)
2210  * \param[in]  nodename  If the object will be used for a remote connection,
2211  *                       the node name to use in cluster for remote executor
2212  * \param[in]  server    If the object will be used for a remote connection,
2213  *                       the resolvable host name to connect to
2214  * \param[in]  port      If the object will be used for a remote connection,
2215  *                       port number on \p server to connect to
2216  *
2217  * \return Standard Pacemaker return code
2218  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2219  *       value will be used for both. If the caller leaves both NULL, an API
2220  *       object will be created for a local executor connection.
2221  */
2222 int
2223 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2224 {
2225     lrmd_private_t *pvt = NULL;
2226 
2227     if (api == NULL) {
2228         return EINVAL;
2229     }
2230     *api = NULL;
2231 
2232     // Allocate all memory needed
2233 
2234     *api = calloc(1, sizeof(lrmd_t));
2235     if (*api == NULL) {
2236         return ENOMEM;
2237     }
2238 
2239     pvt = calloc(1, sizeof(lrmd_private_t));
2240     if (pvt == NULL) {
2241         lrmd_api_delete(*api);
2242         *api = NULL;
2243         return ENOMEM;
2244     }
2245     (*api)->lrmd_private = pvt;
2246 
2247     // @TODO Do we need to do this for local connections?
2248     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2249 
2250     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2251 
2252     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2253         lrmd_api_delete(*api);
2254         *api = NULL;
2255         return ENOMEM;
2256     }
2257 
2258     // Set methods
2259     (*api)->cmds->connect = lrmd_api_connect;
2260     (*api)->cmds->connect_async = lrmd_api_connect_async;
2261     (*api)->cmds->is_connected = lrmd_api_is_connected;
2262     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2263     (*api)->cmds->disconnect = lrmd_api_disconnect;
2264     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2265     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2266     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2267     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2268     (*api)->cmds->set_callback = lrmd_api_set_callback;
2269     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2270     (*api)->cmds->exec = lrmd_api_exec;
2271     (*api)->cmds->cancel = lrmd_api_cancel;
2272     (*api)->cmds->list_agents = lrmd_api_list_agents;
2273     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2274     (*api)->cmds->list_standards = lrmd_api_list_standards;
2275     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2276     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2277 
2278     if ((nodename == NULL) && (server == NULL)) {
2279         pvt->type = pcmk__client_ipc;
2280     } else {
2281 #ifdef HAVE_GNUTLS_GNUTLS_H
2282         if (nodename == NULL) {
2283             nodename = server;
2284         } else if (server == NULL) {
2285             server = nodename;
2286         }
2287         pvt->type = pcmk__client_tls;
2288         pvt->remote_nodename = strdup(nodename);
2289         pvt->server = strdup(server);
2290         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2291             lrmd_api_delete(*api);
2292             *api = NULL;
2293             return ENOMEM;
2294         }
2295         pvt->port = port;
2296         if (pvt->port == 0) {
2297             pvt->port = crm_default_remote_port();
2298         }
2299 #else
2300         crm_err("Cannot communicate with Pacemaker Remote "
2301                 "because GnuTLS is not enabled for this build");
2302         lrmd_api_delete(*api);
2303         *api = NULL;
2304         return EOPNOTSUPP;
2305 #endif
2306     }
2307     return pcmk_rc_ok;
2308 }
2309 
2310 lrmd_t *
2311 lrmd_api_new(void)
     /*  */
2312 {
2313     lrmd_t *api = NULL;
2314 
2315     CRM_ASSERT(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2316     return api;
2317 }
2318 
2319 lrmd_t *
2320 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2321 {
2322     lrmd_t *api = NULL;
2323 
2324     CRM_ASSERT(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2325     return api;
2326 }
2327 
2328 void
2329 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2330 {
2331     if (lrmd == NULL) {
2332         return;
2333     }
2334     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2335         lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2336         free(lrmd->cmds);
2337     }
2338     if (lrmd->lrmd_private != NULL) {
2339         lrmd_private_t *native = lrmd->lrmd_private;
2340 
2341 #ifdef HAVE_GNUTLS_GNUTLS_H
2342         free(native->server);
2343 #endif
2344         free(native->remote_nodename);
2345         free(native->remote);
2346         free(native->token);
2347         free(native->peer_version);
2348         free(lrmd->lrmd_private);
2349     }
2350     free(lrmd);
2351 }
2352 
2353 /*!
2354  * \internal
2355  * \brief Set the result of an executor event
2356  *
2357  * \param[in,out] event        Executor event to set
2358  * \param[in]     rc           OCF exit status of event
2359  * \param[in]     op_status    Executor status of event
2360  * \param[in]     exit_reason  Human-friendly description of event
2361  */
2362 void
2363 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2364                  const char *exit_reason)
2365 {
2366     if (event == NULL) {
2367         return;
2368     }
2369 
2370     event->rc = rc;
2371     event->op_status = op_status;
2372 
2373     if (!pcmk__str_eq(event->exit_reason, exit_reason, pcmk__str_none)) {
2374         free((void *) event->exit_reason);
2375         event->exit_reason = (exit_reason == NULL)? NULL : strdup(exit_reason);
2376     }
2377 }
2378 
2379 /*!
2380  * \internal
2381  * \brief Clear an executor event's exit reason, output, and error output
2382  *
2383  * \param[in] event  Executor event to reset
2384  */
2385 void
2386 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2387 {
2388     if (event == NULL) {
2389         return;
2390     }
2391 
2392     free((void *) event->exit_reason);
2393     event->exit_reason = NULL;
2394 
2395     free((void *) event->output);
2396     event->output = NULL;
2397 }