lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
remote_executor_connected
handle_remote_msg
process_pending_notifies
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd__remote_send_xml
read_remote_reply
send_remote_message
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
lrmd__validate_remote_settings
lrmd_handshake
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
read_gnutls_key
key_is_cached
key_cache_expired
clear_key_cache
get_cached_key
cache_key
get_remote_key
lrmd__init_remote_key
lrmd_gnutls_global_init
tls_handshake_failed
tls_handshake_succeeded
report_async_connection_result
tls_client_handshake
add_tls_to_mainloop
try_handshake_cb
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd__new
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
metadata_complete
lrmd__metadata_async
lrmd__set_result
lrmd__reset_result
lrmd__uptime
lrmd__node_start_state
   1 /*
   2  * Copyright 2012-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/services_internal.h>
  32 #include <crm/common/mainloop.h>
  33 #include <crm/common/ipc_internal.h>
  34 #include <crm/common/remote_internal.h>
  35 #include <crm/common/xml.h>
  36 
  37 #include <crm/stonith-ng.h>
  38 #include <crm/fencing/internal.h>   // stonith__*
  39 
  40 #ifdef HAVE_GNUTLS_GNUTLS_H
  41 #  include <gnutls/gnutls.h>
  42 #endif
  43 
  44 #include <sys/socket.h>
  45 #include <netinet/in.h>
  46 #include <netinet/ip.h>
  47 #include <arpa/inet.h>
  48 #include <netdb.h>
  49 
  50 #define MAX_TLS_RECV_WAIT 10000
  51 
  52 CRM_TRACE_INIT_DATA(lrmd);
  53 
  54 static int lrmd_api_disconnect(lrmd_t * lrmd);
  55 static int lrmd_api_is_connected(lrmd_t * lrmd);
  56 
  57 /* IPC proxy functions */
  58 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  59 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  60 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  61 
  62 #ifdef HAVE_GNUTLS_GNUTLS_H
  63 // GnuTLS client handshake timeout in seconds
  64 #define TLS_HANDSHAKE_TIMEOUT 5
  65 
  66 gnutls_psk_client_credentials_t psk_cred_s;
  67 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  68 static int global_remote_msg_id = 0;
  69 static void lrmd_tls_connection_destroy(gpointer userdata);
  70 static int add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake);
  71 #endif
  72 
  73 static void report_async_connection_result(lrmd_t * lrmd, int rc);
  74 
  75 typedef struct lrmd_private_s {
  76     uint64_t type;
  77     char *token;
  78     mainloop_io_t *source;
  79 
  80     /* IPC parameters */
  81     crm_ipc_t *ipc;
  82 
  83     pcmk__remote_t *remote;
  84 
  85     /* Extra TLS parameters */
  86     char *remote_nodename;
  87 #ifdef HAVE_GNUTLS_GNUTLS_H
  88     char *server;
  89     int port;
  90     gnutls_psk_client_credentials_t psk_cred_c;
  91 
  92     /* while the async connection is occurring, this is the id
  93      * of the connection timeout timer. */
  94     int async_timer;
  95     int sock;
  96     /* since tls requires a round trip across the network for a
  97      * request/reply, there are times where we just want to be able
  98      * to send a request from the client and not wait around (or even care
  99      * about) what the reply is. */
 100     int expected_late_replies;
 101     GList *pending_notify;
 102     crm_trigger_t *process_notify;
 103     crm_trigger_t *handshake_trigger;
 104 #endif
 105 
 106     lrmd_event_callback callback;
 107 
 108     /* Internal IPC proxy msg passing for remote guests */
 109     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 110     void *proxy_callback_userdata;
 111     char *peer_version;
 112 } lrmd_private_t;
 113 
 114 static lrmd_list_t *
 115 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 116 {
 117     lrmd_list_t *p, *end;
 118 
 119     p = pcmk__assert_alloc(1, sizeof(lrmd_list_t));
 120     p->val = strdup(value);
 121 
 122     end = head;
 123     while (end && end->next) {
 124         end = end->next;
 125     }
 126 
 127     if (end) {
 128         end->next = p;
 129     } else {
 130         head = p;
 131     }
 132 
 133     return head;
 134 }
 135 
 136 void
 137 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 138 {
 139     lrmd_list_t *p;
 140 
 141     while (head) {
 142         char *val = (char *)head->val;
 143 
 144         p = head->next;
 145         free(val);
 146         free(head);
 147         head = p;
 148     }
 149 }
 150 
 151 lrmd_key_value_t *
 152 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 153 {
 154     lrmd_key_value_t *p, *end;
 155 
 156     p = pcmk__assert_alloc(1, sizeof(lrmd_key_value_t));
 157     p->key = strdup(key);
 158     p->value = strdup(value);
 159 
 160     end = head;
 161     while (end && end->next) {
 162         end = end->next;
 163     }
 164 
 165     if (end) {
 166         end->next = p;
 167     } else {
 168         head = p;
 169     }
 170 
 171     return head;
 172 }
 173 
 174 void
 175 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 176 {
 177     lrmd_key_value_t *p;
 178 
 179     while (head) {
 180         p = head->next;
 181         free(head->key);
 182         free(head->value);
 183         free(head);
 184         head = p;
 185     }
 186 }
 187 
 188 /*!
 189  * \brief Create a new lrmd_event_data_t object
 190  *
 191  * \param[in] rsc_id       ID of resource involved in event
 192  * \param[in] task         Action name
 193  * \param[in] interval_ms  Action interval
 194  *
 195  * \return Newly allocated and initialized lrmd_event_data_t
 196  * \note This functions asserts on memory errors, so the return value is
 197  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 198  *       result with lrmd_free_event().
 199  */
 200 lrmd_event_data_t *
 201 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 202 {
 203     lrmd_event_data_t *event = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 204 
 205     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 206     event->rsc_id = pcmk__str_copy(rsc_id);
 207     event->op_type = pcmk__str_copy(task);
 208     event->interval_ms = interval_ms;
 209     return event;
 210 }
 211 
 212 lrmd_event_data_t *
 213 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 214 {
 215     lrmd_event_data_t *copy = NULL;
 216 
 217     copy = pcmk__assert_alloc(1, sizeof(lrmd_event_data_t));
 218 
 219     copy->type = event->type;
 220 
 221     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
 222     copy->rsc_id = pcmk__str_copy(event->rsc_id);
 223     copy->op_type = pcmk__str_copy(event->op_type);
 224     copy->user_data = pcmk__str_copy(event->user_data);
 225     copy->output = pcmk__str_copy(event->output);
 226     copy->remote_nodename = pcmk__str_copy(event->remote_nodename);
 227     copy->exit_reason = pcmk__str_copy(event->exit_reason);
 228 
 229     copy->call_id = event->call_id;
 230     copy->timeout = event->timeout;
 231     copy->interval_ms = event->interval_ms;
 232     copy->start_delay = event->start_delay;
 233     copy->rsc_deleted = event->rsc_deleted;
 234     copy->rc = event->rc;
 235     copy->op_status = event->op_status;
 236     copy->t_run = event->t_run;
 237     copy->t_rcchange = event->t_rcchange;
 238     copy->exec_time = event->exec_time;
 239     copy->queue_time = event->queue_time;
 240     copy->connection_rc = event->connection_rc;
 241     copy->params = pcmk__str_table_dup(event->params);
 242 
 243     return copy;
 244 }
 245 
 246 /*!
 247  * \brief Free an executor event
 248  *
 249  * \param[in,out]  Executor event object to free
 250  */
 251 void
 252 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 253 {
 254     if (event == NULL) {
 255         return;
 256     }
 257     // @TODO Why are these const char *?
 258     free((void *) event->rsc_id);
 259     free((void *) event->op_type);
 260     free((void *) event->user_data);
 261     free((void *) event->remote_nodename);
 262     lrmd__reset_result(event);
 263     if (event->params != NULL) {
 264         g_hash_table_destroy(event->params);
 265     }
 266     free(event);
 267 }
 268 
 269 static void
 270 lrmd_dispatch_internal(gpointer data, gpointer user_data)
     /*  */
 271 {
 272     xmlNode *msg = data;
 273     lrmd_t *lrmd = user_data;
 274 
 275     const char *type;
 276     const char *proxy_session = crm_element_value(msg,
 277                                                   PCMK__XA_LRMD_IPC_SESSION);
 278     lrmd_private_t *native = lrmd->lrmd_private;
 279     lrmd_event_data_t event = { 0, };
 280 
 281     if (proxy_session != NULL) {
 282         /* this is proxy business */
 283         lrmd_internal_proxy_dispatch(lrmd, msg);
 284         return;
 285     } else if (!native->callback) {
 286         /* no callback set */
 287         crm_trace("notify event received but client has not set callback");
 288         return;
 289     }
 290 
 291     event.remote_nodename = native->remote_nodename;
 292     type = crm_element_value(msg, PCMK__XA_LRMD_OP);
 293     crm_element_value_int(msg, PCMK__XA_LRMD_CALLID, &event.call_id);
 294     event.rsc_id = crm_element_value(msg, PCMK__XA_LRMD_RSC_ID);
 295 
 296     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 297         event.type = lrmd_event_register;
 298     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 299         event.type = lrmd_event_unregister;
 300     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 301         int rc = 0;
 302         int exec_time = 0;
 303         int queue_time = 0;
 304         time_t epoch = 0;
 305 
 306         crm_element_value_int(msg, PCMK__XA_LRMD_TIMEOUT, &event.timeout);
 307         crm_element_value_ms(msg, PCMK__XA_LRMD_RSC_INTERVAL,
 308                              &event.interval_ms);
 309         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_START_DELAY,
 310                               &event.start_delay);
 311 
 312         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_RC, &rc);
 313         event.rc = (enum ocf_exitcode) rc;
 314 
 315         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_OP_STATUS,
 316                               &event.op_status);
 317         crm_element_value_int(msg, PCMK__XA_LRMD_RSC_DELETED,
 318                               &event.rsc_deleted);
 319 
 320         crm_element_value_epoch(msg, PCMK__XA_LRMD_RUN_TIME, &epoch);
 321         // coverity[store_truncates_time_t]
 322         event.t_run = (unsigned int) epoch;
 323 
 324         crm_element_value_epoch(msg, PCMK__XA_LRMD_RCCHANGE_TIME, &epoch);
 325         // coverity[store_truncates_time_t]
 326         event.t_rcchange = (unsigned int) epoch;
 327 
 328         crm_element_value_int(msg, PCMK__XA_LRMD_EXEC_TIME, &exec_time);
 329         CRM_LOG_ASSERT(exec_time >= 0);
 330         event.exec_time = QB_MAX(0, exec_time);
 331 
 332         crm_element_value_int(msg, PCMK__XA_LRMD_QUEUE_TIME, &queue_time);
 333         CRM_LOG_ASSERT(queue_time >= 0);
 334         event.queue_time = QB_MAX(0, queue_time);
 335 
 336         event.op_type = crm_element_value(msg, PCMK__XA_LRMD_RSC_ACTION);
 337         event.user_data = crm_element_value(msg,
 338                                             PCMK__XA_LRMD_RSC_USERDATA_STR);
 339         event.type = lrmd_event_exec_complete;
 340 
 341         /* output and exit_reason may be freed by a callback */
 342         event.output = crm_element_value_copy(msg, PCMK__XA_LRMD_RSC_OUTPUT);
 343         lrmd__set_result(&event, event.rc, event.op_status,
 344                          crm_element_value(msg, PCMK__XA_LRMD_RSC_EXIT_REASON));
 345 
 346         event.params = xml2list(msg);
 347     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 348         event.type = lrmd_event_new_client;
 349     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 350         event.type = lrmd_event_poke;
 351     } else {
 352         return;
 353     }
 354 
 355     crm_trace("op %s notify event received", type);
 356     native->callback(&event);
 357 
 358     if (event.params) {
 359         g_hash_table_destroy(event.params);
 360     }
 361     lrmd__reset_result(&event);
 362 }
 363 
 364 // \return Always 0, to indicate that IPC mainloop source should be kept
 365 static int
 366 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 367 {
 368     lrmd_t *lrmd = userdata;
 369     lrmd_private_t *native = lrmd->lrmd_private;
 370 
 371     if (native->callback != NULL) {
 372         xmlNode *msg = pcmk__xml_parse(buffer);
 373 
 374         lrmd_dispatch_internal(msg, lrmd);
 375         free_xml(msg);
 376     }
 377     return 0;
 378 }
 379 
 380 #ifdef HAVE_GNUTLS_GNUTLS_H
 381 static void
 382 lrmd_free_xml(gpointer userdata)
     /*  */
 383 {
 384     free_xml((xmlNode *) userdata);
 385 }
 386 
 387 static bool
 388 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 389 {
 390     lrmd_private_t *native = lrmd->lrmd_private;
 391 
 392     return (native->remote->tls_session != NULL);
 393 }
 394 
 395 static void
 396 handle_remote_msg(xmlNode *xml, lrmd_t *lrmd)
     /*  */
 397 {
 398     lrmd_private_t *native = lrmd->lrmd_private;
 399     const char *msg_type = NULL;
 400 
 401     msg_type = crm_element_value(xml, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 402     if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 403         lrmd_dispatch_internal(xml, lrmd);
 404     } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 405         if (native->expected_late_replies > 0) {
 406             native->expected_late_replies--;
 407         } else {
 408             int reply_id = 0;
 409             crm_element_value_int(xml, PCMK__XA_LRMD_CALLID, &reply_id);
 410             /* if this happens, we want to know about it */
 411             crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 412         }
 413     }
 414 }
 415 
 416 static void
 417 process_pending_notifies(lrmd_t *lrmd)
     /*  */
 418 {
 419     lrmd_private_t *native = lrmd->lrmd_private;
 420 
 421     if (native->pending_notify == NULL) {
 422         return;
 423     }
 424 
 425     crm_trace("Processing pending notifies");
 426     g_list_foreach(native->pending_notify, lrmd_dispatch_internal, lrmd);
 427     g_list_free_full(native->pending_notify, lrmd_free_xml);
 428     native->pending_notify = NULL;
 429 }
 430 
 431 /*!
 432  * \internal
 433  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 434  *
 435  * \param[in,out] userdata  API connection
 436  *
 437  * \return Always return a nonnegative value, which as a file descriptor
 438  *         dispatch function means keep the mainloop source, and as a
 439  *         trigger dispatch function, 0 means remove the trigger from the
 440  *         mainloop while 1 means keep it (and job completed)
 441  */
 442 static int
 443 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 444 {
 445     lrmd_t *lrmd = userdata;
 446     lrmd_private_t *native = lrmd->lrmd_private;
 447     xmlNode *xml = NULL;
 448     int rc = pcmk_rc_ok;
 449 
 450     if (!remote_executor_connected(lrmd)) {
 451         crm_trace("TLS dispatch triggered after disconnect");
 452         return 0;
 453     }
 454 
 455     crm_trace("TLS dispatch triggered");
 456 
 457     /* First check if there are any pending notifies to process that came
 458      * while we were waiting for replies earlier.
 459      */
 460     process_pending_notifies(lrmd);
 461 
 462     /* Next read the current buffer and see if there are any messages to handle. */
 463     rc = pcmk__remote_ready(native->remote, 0);
 464     if (rc == pcmk_rc_ok) {
 465         rc = pcmk__read_remote_message(native->remote, -1);
 466     }
 467 
 468     if (rc != pcmk_rc_ok && rc != ETIME) {
 469         crm_info("Lost %s executor connection while reading data",
 470                  (native->remote_nodename? native->remote_nodename : "local"));
 471         lrmd_tls_disconnect(lrmd);
 472         return 0;
 473     }
 474 
 475     /* If rc is ETIME, there was nothing to read but we may already have a
 476      * full message in the buffer
 477      */
 478     xml = pcmk__remote_message_xml(native->remote);
 479 
 480     if (xml == NULL) {
 481         return 1;
 482     }
 483 
 484     handle_remote_msg(xml, lrmd);
 485     free_xml(xml);
 486     return 1;
 487 }
 488 #endif
 489 
 490 /* Not used with mainloop */
 491 int
 492 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 493 {
 494     lrmd_private_t *native = lrmd->lrmd_private;
 495 
 496     switch (native->type) {
 497         case pcmk__client_ipc:
 498             return crm_ipc_ready(native->ipc);
 499 
 500 #ifdef HAVE_GNUTLS_GNUTLS_H
 501         case pcmk__client_tls:
 502             if (native->pending_notify) {
 503                 return 1;
 504             } else {
 505                 int rc = pcmk__remote_ready(native->remote, 0);
 506 
 507                 switch (rc) {
 508                     case pcmk_rc_ok:
 509                         return 1;
 510                     case ETIME:
 511                         return 0;
 512                     default:
 513                         return pcmk_rc2legacy(rc);
 514                 }
 515             }
 516 #endif
 517         default:
 518             crm_err("Unsupported executor connection type (bug?): %d",
 519                     native->type);
 520             return -EPROTONOSUPPORT;
 521     }
 522 }
 523 
 524 /* Not used with mainloop */
 525 bool
 526 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 527 {
 528     lrmd_private_t *private = NULL;
 529 
 530     pcmk__assert(lrmd != NULL);
 531 
 532     private = lrmd->lrmd_private;
 533     switch (private->type) {
 534         case pcmk__client_ipc:
 535             while (crm_ipc_ready(private->ipc)) {
 536                 if (crm_ipc_read(private->ipc) > 0) {
 537                     const char *msg = crm_ipc_buffer(private->ipc);
 538 
 539                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 540                 }
 541             }
 542             break;
 543 #ifdef HAVE_GNUTLS_GNUTLS_H
 544         case pcmk__client_tls:
 545             lrmd_tls_dispatch(lrmd);
 546             break;
 547 #endif
 548         default:
 549             crm_err("Unsupported executor connection type (bug?): %d",
 550                     private->type);
 551     }
 552 
 553     if (lrmd_api_is_connected(lrmd) == FALSE) {
 554         crm_err("Connection closed");
 555         return FALSE;
 556     }
 557 
 558     return TRUE;
 559 }
 560 
 561 static xmlNode *
 562 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 563                enum lrmd_call_options options)
 564 {
 565     xmlNode *op_msg = NULL;
 566 
 567     CRM_CHECK(token != NULL, return NULL);
 568 
 569     op_msg = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
 570     crm_xml_add(op_msg, PCMK__XA_T, PCMK__VALUE_LRMD);
 571     crm_xml_add(op_msg, PCMK__XA_LRMD_OP, op);
 572     crm_xml_add_int(op_msg, PCMK__XA_LRMD_TIMEOUT, timeout);
 573     crm_xml_add_int(op_msg, PCMK__XA_LRMD_CALLOPT, options);
 574 
 575     if (data != NULL) {
 576         xmlNode *wrapper = pcmk__xe_create(op_msg, PCMK__XE_LRMD_CALLDATA);
 577 
 578         pcmk__xml_copy(wrapper, data);
 579     }
 580 
 581     crm_trace("Created executor %s command with call options %.8lx (%d)",
 582               op, (long)options, options);
 583     return op_msg;
 584 }
 585 
 586 static void
 587 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 588 {
 589     lrmd_t *lrmd = userdata;
 590     lrmd_private_t *native = lrmd->lrmd_private;
 591 
 592     switch (native->type) {
 593         case pcmk__client_ipc:
 594             crm_info("Disconnected from local executor");
 595             break;
 596 #ifdef HAVE_GNUTLS_GNUTLS_H
 597         case pcmk__client_tls:
 598             crm_info("Disconnected from remote executor on %s",
 599                      native->remote_nodename);
 600             break;
 601 #endif
 602         default:
 603             crm_err("Unsupported executor connection type %d (bug?)",
 604                     native->type);
 605     }
 606 
 607     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 608     native->ipc = NULL;
 609     native->source = NULL;
 610 
 611     if (native->callback) {
 612         lrmd_event_data_t event = { 0, };
 613         event.type = lrmd_event_disconnect;
 614         event.remote_nodename = native->remote_nodename;
 615         native->callback(&event);
 616     }
 617 }
 618 
 619 #ifdef HAVE_GNUTLS_GNUTLS_H
 620 static void
 621 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 622 {
 623     lrmd_t *lrmd = userdata;
 624     lrmd_private_t *native = lrmd->lrmd_private;
 625 
 626     crm_info("TLS connection destroyed");
 627 
 628     if (native->remote->tls_session) {
 629         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 630         gnutls_deinit(*native->remote->tls_session);
 631         gnutls_free(native->remote->tls_session);
 632         native->remote->tls_session = NULL;
 633     }
 634     if (native->psk_cred_c) {
 635         gnutls_psk_free_client_credentials(native->psk_cred_c);
 636     }
 637     if (native->sock) {
 638         close(native->sock);
 639     }
 640     if (native->process_notify) {
 641         mainloop_destroy_trigger(native->process_notify);
 642         native->process_notify = NULL;
 643     }
 644     if (native->pending_notify) {
 645         g_list_free_full(native->pending_notify, lrmd_free_xml);
 646         native->pending_notify = NULL;
 647     }
 648     if (native->handshake_trigger != NULL) {
 649         mainloop_destroy_trigger(native->handshake_trigger);
 650         native->handshake_trigger = NULL;
 651     }
 652 
 653     free(native->remote->buffer);
 654     free(native->remote->start_state);
 655     native->remote->buffer = NULL;
 656     native->remote->start_state = NULL;
 657     native->source = 0;
 658     native->sock = 0;
 659     native->psk_cred_c = NULL;
 660     native->sock = 0;
 661 
 662     if (native->callback) {
 663         lrmd_event_data_t event = { 0, };
 664         event.remote_nodename = native->remote_nodename;
 665         event.type = lrmd_event_disconnect;
 666         native->callback(&event);
 667     }
 668     return;
 669 }
 670 
 671 // \return Standard Pacemaker return code
 672 int
 673 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 674                       const char *msg_type)
 675 {
 676     crm_xml_add_int(msg, PCMK__XA_LRMD_REMOTE_MSG_ID, id);
 677     crm_xml_add(msg, PCMK__XA_LRMD_REMOTE_MSG_TYPE, msg_type);
 678     return pcmk__remote_send_xml(session, msg);
 679 }
 680 
 681 // \return Standard Pacemaker return code
 682 static int
 683 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 684                   xmlNode **reply)
 685 {
 686     lrmd_private_t *native = lrmd->lrmd_private;
 687     time_t start = time(NULL);
 688     const char *msg_type = NULL;
 689     int reply_id = 0;
 690     int remaining_timeout = 0;
 691     int rc = pcmk_rc_ok;
 692 
 693     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 694      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 695     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 696         total_timeout = MAX_TLS_RECV_WAIT;
 697     }
 698 
 699     for (*reply = NULL; *reply == NULL; ) {
 700 
 701         *reply = pcmk__remote_message_xml(native->remote);
 702         if (*reply == NULL) {
 703             /* read some more off the tls buffer if we still have time left. */
 704             if (remaining_timeout) {
 705                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 706             } else {
 707                 remaining_timeout = total_timeout;
 708             }
 709             if (remaining_timeout <= 0) {
 710                 return ETIME;
 711             }
 712 
 713             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 714             if (rc != pcmk_rc_ok) {
 715                 return rc;
 716             }
 717 
 718             *reply = pcmk__remote_message_xml(native->remote);
 719             if (*reply == NULL) {
 720                 return ENOMSG;
 721             }
 722         }
 723 
 724         crm_element_value_int(*reply, PCMK__XA_LRMD_REMOTE_MSG_ID, &reply_id);
 725         msg_type = crm_element_value(*reply, PCMK__XA_LRMD_REMOTE_MSG_TYPE);
 726 
 727         if (!msg_type) {
 728             crm_err("Empty msg type received while waiting for reply");
 729             free_xml(*reply);
 730             *reply = NULL;
 731         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 732             /* got a notify while waiting for reply, trigger the notify to be processed later */
 733             crm_info("queueing notify");
 734             native->pending_notify = g_list_append(native->pending_notify, *reply);
 735             if (native->process_notify) {
 736                 crm_info("notify trigger set.");
 737                 mainloop_set_trigger(native->process_notify);
 738             }
 739             *reply = NULL;
 740         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 741             /* msg isn't a reply, make some noise */
 742             crm_err("Expected a reply, got %s", msg_type);
 743             free_xml(*reply);
 744             *reply = NULL;
 745         } else if (reply_id != expected_reply_id) {
 746             if (native->expected_late_replies > 0) {
 747                 native->expected_late_replies--;
 748             } else {
 749                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 750             }
 751             free_xml(*reply);
 752             *reply = NULL;
 753         }
 754     }
 755 
 756     if (native->remote->buffer && native->process_notify) {
 757         mainloop_set_trigger(native->process_notify);
 758     }
 759 
 760     return rc;
 761 }
 762 
 763 // \return Standard Pacemaker return code
 764 static int
 765 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 766 {
 767     int rc = pcmk_rc_ok;
 768     lrmd_private_t *native = lrmd->lrmd_private;
 769 
 770     global_remote_msg_id++;
 771     if (global_remote_msg_id <= 0) {
 772         global_remote_msg_id = 1;
 773     }
 774 
 775     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 776                                "request");
 777     if (rc != pcmk_rc_ok) {
 778         crm_err("Disconnecting because TLS message could not be sent to "
 779                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 780         lrmd_tls_disconnect(lrmd);
 781     }
 782     return rc;
 783 }
 784 
 785 static int
 786 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 787 {
 788     int rc = 0;
 789     xmlNode *xml = NULL;
 790 
 791     if (!remote_executor_connected(lrmd)) {
 792         return -ENOTCONN;
 793     }
 794 
 795     rc = send_remote_message(lrmd, msg);
 796     if (rc != pcmk_rc_ok) {
 797         return pcmk_rc2legacy(rc);
 798     }
 799 
 800     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 801     if (rc != pcmk_rc_ok) {
 802         crm_err("Disconnecting remote after request %d reply not received: %s "
 803                 CRM_XS " rc=%d timeout=%dms",
 804                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 805         lrmd_tls_disconnect(lrmd);
 806     }
 807 
 808     if (reply) {
 809         *reply = xml;
 810     } else {
 811         free_xml(xml);
 812     }
 813 
 814     return pcmk_rc2legacy(rc);
 815 }
 816 #endif
 817 
 818 static int
 819 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 820 {
 821     int rc = pcmk_ok;
 822     lrmd_private_t *native = lrmd->lrmd_private;
 823 
 824     switch (native->type) {
 825         case pcmk__client_ipc:
 826             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 827             break;
 828 #ifdef HAVE_GNUTLS_GNUTLS_H
 829         case pcmk__client_tls:
 830             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 831             break;
 832 #endif
 833         default:
 834             crm_err("Unsupported executor connection type (bug?): %d",
 835                     native->type);
 836             rc = -EPROTONOSUPPORT;
 837     }
 838 
 839     return rc;
 840 }
 841 
 842 static int
 843 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 844 {
 845     int rc = pcmk_ok;
 846     lrmd_private_t *native = lrmd->lrmd_private;
 847 
 848     switch (native->type) {
 849         case pcmk__client_ipc:
 850             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 851             break;
 852 #ifdef HAVE_GNUTLS_GNUTLS_H
 853         case pcmk__client_tls:
 854             rc = send_remote_message(lrmd, msg);
 855             if (rc == pcmk_rc_ok) {
 856                 /* we don't want to wait around for the reply, but
 857                  * since the request/reply protocol needs to behave the same
 858                  * as libqb, a reply will eventually come later anyway. */
 859                 native->expected_late_replies++;
 860             }
 861             rc = pcmk_rc2legacy(rc);
 862             break;
 863 #endif
 864         default:
 865             crm_err("Unsupported executor connection type (bug?): %d",
 866                     native->type);
 867             rc = -EPROTONOSUPPORT;
 868     }
 869 
 870     return rc;
 871 }
 872 
 873 static int
 874 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 875 {
 876     lrmd_private_t *native = lrmd->lrmd_private;
 877 
 878     switch (native->type) {
 879         case pcmk__client_ipc:
 880             return crm_ipc_connected(native->ipc);
 881 #ifdef HAVE_GNUTLS_GNUTLS_H
 882         case pcmk__client_tls:
 883             return remote_executor_connected(lrmd);
 884 #endif
 885         default:
 886             crm_err("Unsupported executor connection type (bug?): %d",
 887                     native->type);
 888             return 0;
 889     }
 890 }
 891 
 892 /*!
 893  * \internal
 894  * \brief Send a prepared API command to the executor
 895  *
 896  * \param[in,out] lrmd          Existing connection to the executor
 897  * \param[in]     op            Name of API command to send
 898  * \param[in]     data          Command data XML to add to the sent command
 899  * \param[out]    output_data   If expecting a reply, it will be stored here
 900  * \param[in]     timeout       Timeout in milliseconds (if 0, defaults to
 901  *                              a sensible value per the type of connection,
 902  *                              standard vs. pacemaker remote);
 903  *                              also propagated to the command XML
 904  * \param[in]     call_options  Call options to pass to server when sending
 905  * \param[in]     expect_reply  If TRUE, wait for a reply from the server;
 906  *                              must be TRUE for IPC (as opposed to TLS) clients
 907  *
 908  * \return pcmk_ok on success, -errno on error
 909  */
 910 static int
 911 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 912                   xmlNode **output_data, int timeout,
 913                   enum lrmd_call_options options, gboolean expect_reply)
 914 {
 915     int rc = pcmk_ok;
 916     lrmd_private_t *native = lrmd->lrmd_private;
 917     xmlNode *op_msg = NULL;
 918     xmlNode *op_reply = NULL;
 919 
 920     if (!lrmd_api_is_connected(lrmd)) {
 921         return -ENOTCONN;
 922     }
 923 
 924     if (op == NULL) {
 925         crm_err("No operation specified");
 926         return -EINVAL;
 927     }
 928 
 929     CRM_CHECK(native->token != NULL,;
 930         );
 931     crm_trace("Sending %s op to executor", op);
 932 
 933     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 934 
 935     if (op_msg == NULL) {
 936         return -EINVAL;
 937     }
 938 
 939     if (expect_reply) {
 940         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 941     } else {
 942         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 943         goto done;
 944     }
 945 
 946     if (rc < 0) {
 947         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 948         goto done;
 949 
 950     } else if(op_reply == NULL) {
 951         rc = -ENOMSG;
 952         goto done;
 953     }
 954 
 955     rc = pcmk_ok;
 956     crm_trace("%s op reply received", op);
 957     if (crm_element_value_int(op_reply, PCMK__XA_LRMD_RC, &rc) != 0) {
 958         rc = -ENOMSG;
 959         goto done;
 960     }
 961 
 962     crm_log_xml_trace(op_reply, "Reply");
 963 
 964     if (output_data) {
 965         *output_data = op_reply;
 966         op_reply = NULL;        /* Prevent subsequent free */
 967     }
 968 
 969   done:
 970     if (lrmd_api_is_connected(lrmd) == FALSE) {
 971         crm_err("Executor disconnected");
 972     }
 973 
 974     free_xml(op_msg);
 975     free_xml(op_reply);
 976     return rc;
 977 }
 978 
 979 static int
 980 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 981 {
 982     int rc;
 983     lrmd_private_t *native = lrmd->lrmd_private;
 984     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
 985 
 986     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
 987     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 988                            (native->type == pcmk__client_ipc));
 989     free_xml(data);
 990 
 991     return rc < 0 ? rc : pcmk_ok;
 992 }
 993 
 994 // \return Standard Pacemaker return code
 995 int
 996 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 997 {
 998     int rc = pcmk_rc_ok;
 999     const char *value;
1000     lrmd_private_t *native = lrmd->lrmd_private;
1001     xmlNode *data = pcmk__xe_create(NULL, PCMK__XA_LRMD_OP);
1002 
1003     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1004 
1005     value = g_hash_table_lookup(hash, PCMK_OPT_STONITH_WATCHDOG_TIMEOUT);
1006     if ((value) &&
1007         (stonith__watchdog_fencing_enabled_for_node(native->remote_nodename))) {
1008        crm_xml_add(data, PCMK__XA_LRMD_WATCHDOG, value);
1009     }
1010 
1011     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
1012                            (native->type == pcmk__client_ipc));
1013     free_xml(data);
1014     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
1015 }
1016 
1017 static int
1018 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
1019 {
1020     int rc = pcmk_ok;
1021     lrmd_private_t *native = lrmd->lrmd_private;
1022     xmlNode *reply = NULL;
1023     xmlNode *hello = pcmk__xe_create(NULL, PCMK__XE_LRMD_COMMAND);
1024 
1025     crm_xml_add(hello, PCMK__XA_T, PCMK__VALUE_LRMD);
1026     crm_xml_add(hello, PCMK__XA_LRMD_OP, CRM_OP_REGISTER);
1027     crm_xml_add(hello, PCMK__XA_LRMD_CLIENTNAME, name);
1028     crm_xml_add(hello, PCMK__XA_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
1029 
1030     /* advertise that we are a proxy provider */
1031     if (native->proxy_callback) {
1032         pcmk__xe_set_bool_attr(hello, PCMK__XA_LRMD_IS_IPC_PROVIDER, true);
1033     }
1034 
1035     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
1036 
1037     if (rc < 0) {
1038         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
1039         rc = -ECOMM;
1040     } else if (reply == NULL) {
1041         crm_err("Did not receive registration reply");
1042         rc = -EPROTO;
1043     } else {
1044         const char *version = crm_element_value(reply,
1045                                                 PCMK__XA_LRMD_PROTOCOL_VERSION);
1046         const char *msg_type = crm_element_value(reply, PCMK__XA_LRMD_OP);
1047         const char *tmp_ticket = crm_element_value(reply,
1048                                                    PCMK__XA_LRMD_CLIENTID);
1049         const char *start_state = crm_element_value(reply, PCMK__XA_NODE_START_STATE);
1050         long long uptime = -1;
1051 
1052         crm_element_value_int(reply, PCMK__XA_LRMD_RC, &rc);
1053 
1054         /* The remote executor may add its uptime to the XML reply, which is
1055          * useful in handling transient attributes when the connection to the
1056          * remote node unexpectedly drops.  If no parameter is given, just
1057          * default to -1.
1058          */
1059         crm_element_value_ll(reply, PCMK__XA_UPTIME, &uptime);
1060         native->remote->uptime = uptime;
1061 
1062         if (start_state) {
1063             native->remote->start_state = strdup(start_state);
1064         }
1065 
1066         if (rc == -EPROTO) {
1067             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
1068                 LRMD_PROTOCOL_VERSION, version);
1069             crm_log_xml_err(reply, "Protocol Error");
1070 
1071         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
1072             crm_err("Invalid registration message: %s", msg_type);
1073             crm_log_xml_err(reply, "Bad reply");
1074             rc = -EPROTO;
1075         } else if (tmp_ticket == NULL) {
1076             crm_err("No registration token provided");
1077             crm_log_xml_err(reply, "Bad reply");
1078             rc = -EPROTO;
1079         } else {
1080             crm_trace("Obtained registration token: %s", tmp_ticket);
1081             native->token = strdup(tmp_ticket);
1082             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
1083             rc = pcmk_ok;
1084         }
1085     }
1086 
1087     free_xml(reply);
1088     free_xml(hello);
1089 
1090     if (rc != pcmk_ok) {
1091         lrmd_api_disconnect(lrmd);
1092     }
1093     return rc;
1094 }
1095 
1096 static int
1097 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1098 {
1099     int rc = pcmk_ok;
1100     lrmd_private_t *native = lrmd->lrmd_private;
1101 
1102     struct ipc_client_callbacks lrmd_callbacks = {
1103         .dispatch = lrmd_ipc_dispatch,
1104         .destroy = lrmd_ipc_connection_destroy
1105     };
1106 
1107     crm_info("Connecting to executor");
1108 
1109     if (fd) {
1110         /* No mainloop */
1111         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1112         if (native->ipc != NULL) {
1113             rc = pcmk__connect_generic_ipc(native->ipc);
1114             if (rc == pcmk_rc_ok) {
1115                 rc = pcmk__ipc_fd(native->ipc, fd);
1116             }
1117             if (rc != pcmk_rc_ok) {
1118                 crm_err("Connection to executor failed: %s", pcmk_rc_str(rc));
1119                 rc = -ENOTCONN;
1120             }
1121         }
1122     } else {
1123         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1124         native->ipc = mainloop_get_ipc_client(native->source);
1125     }
1126 
1127     if (native->ipc == NULL) {
1128         crm_debug("Could not connect to the executor API");
1129         rc = -ENOTCONN;
1130     }
1131 
1132     return rc;
1133 }
1134 
1135 #ifdef HAVE_GNUTLS_GNUTLS_H
1136 static void
1137 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1138 {
1139     pcmk__assert((dest != NULL) && (source != NULL) && (source->data != NULL));
1140 
1141     dest->data = gnutls_malloc(source->size);
1142     pcmk__mem_assert(dest->data);
1143 
1144     memcpy(dest->data, source->data, source->size);
1145     dest->size = source->size;
1146 }
1147 
1148 static void
1149 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1150 {
1151     gnutls_free(datum->data);
1152     datum->data = NULL;
1153     datum->size = 0;
1154 }
1155 
1156 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1157 
1158 // \return Standard Pacemaker return code
1159 static int
1160 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1161 {
1162     FILE *stream = NULL;
1163     size_t buf_len = KEY_READ_LEN;
1164 
1165     if ((location == NULL) || (key == NULL)) {
1166         return EINVAL;
1167     }
1168 
1169     stream = fopen(location, "r");
1170     if (stream == NULL) {
1171         return errno;
1172     }
1173 
1174     key->data = gnutls_malloc(buf_len);
1175     key->size = 0;
1176     while (!feof(stream)) {
1177         int next = fgetc(stream);
1178 
1179         if (next == EOF) {
1180             if (!feof(stream)) {
1181                 crm_warn("Pacemaker Remote key read was partially successful "
1182                          "(copy in memory may be corrupted)");
1183             }
1184             break;
1185         }
1186         if (key->size == buf_len) {
1187             buf_len = key->size + KEY_READ_LEN;
1188             key->data = gnutls_realloc(key->data, buf_len);
1189             pcmk__assert(key->data);
1190         }
1191         key->data[key->size++] = (unsigned char) next;
1192     }
1193     fclose(stream);
1194 
1195     if (key->size == 0) {
1196         clear_gnutls_datum(key);
1197         return ENOKEY;
1198     }
1199     return pcmk_rc_ok;
1200 }
1201 
1202 // Cache the most recently used Pacemaker Remote authentication key
1203 
1204 struct key_cache_s {
1205     time_t updated;         // When cached key was read (valid for 1 minute)
1206     const char *location;   // Where cached key was read from
1207     gnutls_datum_t key;     // Cached key
1208 };
1209 
1210 static bool
1211 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1212 {
1213     return key_cache->updated != 0;
1214 }
1215 
1216 static bool
1217 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1218 {
1219     return (time(NULL) - key_cache->updated) >= 60;
1220 }
1221 
1222 static void
1223 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1224 {
1225     clear_gnutls_datum(&(key_cache->key));
1226     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1227         key_cache->updated = 0;
1228         key_cache->location = NULL;
1229         crm_debug("Cleared Pacemaker Remote key cache");
1230     }
1231 }
1232 
1233 static void
1234 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1235 {
1236     copy_gnutls_datum(key, &(key_cache->key));
1237     crm_debug("Using cached Pacemaker Remote key from %s",
1238               pcmk__s(key_cache->location, "unknown location"));
1239 }
1240 
1241 static void
1242 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1243           const char *location)
1244 {
1245     key_cache->updated = time(NULL);
1246     key_cache->location = location;
1247     copy_gnutls_datum(&(key_cache->key), key);
1248     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1249               pcmk__s(location, "unknown location"));
1250 }
1251 
1252 /*!
1253  * \internal
1254  * \brief Get Pacemaker Remote authentication key from file or cache
1255  *
1256  * \param[in]  location         Path to key file to try (this memory must
1257  *                              persist across all calls of this function)
1258  * \param[out] key              Key from location or cache
1259  *
1260  * \return Standard Pacemaker return code
1261  */
1262 static int
1263 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1264 {
1265     static struct key_cache_s key_cache = { 0, };
1266     int rc = pcmk_rc_ok;
1267 
1268     if ((location == NULL) || (key == NULL)) {
1269         return EINVAL;
1270     }
1271 
1272     if (key_is_cached(&key_cache)) {
1273         if (key_cache_expired(&key_cache)) {
1274             clear_key_cache(&key_cache);
1275         } else {
1276             get_cached_key(&key_cache, key);
1277             return pcmk_rc_ok;
1278         }
1279     }
1280 
1281     rc = read_gnutls_key(location, key);
1282     if (rc != pcmk_rc_ok) {
1283         return rc;
1284     }
1285     cache_key(&key_cache, key, location);
1286     return pcmk_rc_ok;
1287 }
1288 
1289 /*!
1290  * \internal
1291  * \brief Initialize the Pacemaker Remote authentication key
1292  *
1293  * Try loading the Pacemaker Remote authentication key from cache if available,
1294  * otherwise from these locations, in order of preference: the value of the
1295  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1296  * file location; or (for historical reasons) /etc/corosync/authkey.
1297  *
1298  * \param[out] key  Where to store key
1299  *
1300  * \return Standard Pacemaker return code
1301  */
1302 int
1303 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1304 {
1305     static const char *env_location = NULL;
1306     static bool need_env = true;
1307 
1308     int env_rc = pcmk_rc_ok;
1309     int default_rc = pcmk_rc_ok;
1310     int alt_rc = pcmk_rc_ok;
1311 
1312     bool env_is_default = false;
1313     bool env_is_fallback = false;
1314 
1315     if (need_env) {
1316         env_location = pcmk__env_option(PCMK__ENV_AUTHKEY_LOCATION);
1317         need_env = false;
1318     }
1319 
1320     // Try location in environment variable, if set
1321     if (env_location != NULL) {
1322         env_rc = get_remote_key(env_location, key);
1323         if (env_rc == pcmk_rc_ok) {
1324             return pcmk_rc_ok;
1325         }
1326 
1327         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1328         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1329 
1330         /* @TODO It would be more secure to fail, rather than fall back to the
1331          * default, if an explicitly set key location is not readable, and it
1332          * would be better to never use the Corosync location as a fallback.
1333          * However, that would break any deployments currently working with the
1334          * fallbacks.
1335          *
1336          * @COMPAT Change at 3.0.0
1337          */
1338     }
1339 
1340     // Try default location, if environment wasn't explicitly set to it
1341     if (env_is_default) {
1342         default_rc = env_rc;
1343     } else {
1344         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1345     }
1346 
1347     // Try fallback location, if environment wasn't set to it and default failed
1348     // @COMPAT Drop at 3.0.0
1349     if (env_is_fallback) {
1350         alt_rc = env_rc;
1351     } else if (default_rc != pcmk_rc_ok) {
1352         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1353     }
1354 
1355     // We have all results, so log and return
1356 
1357     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1358         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1359 
1360         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1361                  env_location,
1362                  env_is_default? "" : "or default location ",
1363                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1364                  !env_is_default && !env_is_fallback? " " : "",
1365                  env_is_fallback? "" : "or fallback location ",
1366                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1367                  pcmk_rc_str(env_rc));
1368         return ENOKEY;
1369     }
1370 
1371     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1372         crm_warn("Could not read Pacemaker Remote key from %s "
1373                  "(using %s location %s instead): %s",
1374                  env_location,
1375                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1376                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1377                  pcmk_rc_str(env_rc));
1378         crm_warn("This undocumented behavior is deprecated and unsafe and will "
1379                  "be removed in a future release");
1380         return pcmk_rc_ok;
1381     }
1382 
1383     if (default_rc != pcmk_rc_ok) {
1384         if (alt_rc == pcmk_rc_ok) {
1385             // Environment variable unset, used alternate location
1386             // This gets caught by the default return below, but we additionally
1387             // warn on this behavior here.
1388             crm_warn("Read Pacemaker Remote key from alternate location %s",
1389                      ALT_REMOTE_KEY_LOCATION);
1390             crm_warn("This undocumented behavior is deprecated and unsafe and will "
1391                      "be removed in a future release");
1392         } else {
1393             // Environment unset, defaults failed
1394             crm_warn("Could not read Pacemaker Remote key from default location %s"
1395                      " (or fallback location %s): %s",
1396                      DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1397                      pcmk_rc_str(default_rc));
1398             return ENOKEY;
1399         }
1400     }
1401 
1402     return pcmk_rc_ok; // Environment variable unset, a default worked
1403 }
1404 
1405 static void
1406 lrmd_gnutls_global_init(void)
     /*  */
1407 {
1408     static int gnutls_init = 0;
1409 
1410     if (!gnutls_init) {
1411         crm_gnutls_global_init();
1412     }
1413     gnutls_init = 1;
1414 }
1415 
1416 static void
1417 tls_handshake_failed(lrmd_t *lrmd, int tls_rc, int rc)
     /*  */
1418 {
1419     lrmd_private_t *native = lrmd->lrmd_private;
1420 
1421     crm_warn("Disconnecting after TLS handshake with "
1422              "Pacemaker Remote server %s:%d failed: %s",
1423              native->server, native->port,
1424              (rc == EPROTO)? gnutls_strerror(tls_rc) : pcmk_rc_str(rc));
1425     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1426 
1427     gnutls_deinit(*native->remote->tls_session);
1428     gnutls_free(native->remote->tls_session);
1429     native->remote->tls_session = NULL;
1430     lrmd_tls_connection_destroy(lrmd);
1431 }
1432 
1433 static void
1434 tls_handshake_succeeded(lrmd_t *lrmd)
     /*  */
1435 {
1436     int rc = pcmk_rc_ok;
1437     lrmd_private_t *native = lrmd->lrmd_private;
1438 
1439     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1440              native->server, native->port);
1441     rc = add_tls_to_mainloop(lrmd, true);
1442     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1443 }
1444 #endif
1445 
1446 static void
1447 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1448 {
1449     lrmd_private_t *native = lrmd->lrmd_private;
1450 
1451     if (native->callback) {
1452         lrmd_event_data_t event = { 0, };
1453         event.type = lrmd_event_connect;
1454         event.remote_nodename = native->remote_nodename;
1455         event.connection_rc = rc;
1456         native->callback(&event);
1457     }
1458 }
1459 
1460 #ifdef HAVE_GNUTLS_GNUTLS_H
1461 /*!
1462  * \internal
1463  * \brief Perform a TLS client handshake with a Pacemaker Remote server
1464  *
1465  * \param[in] lrmd  Newly established Pacemaker Remote executor connection
1466  *
1467  * \return Standard Pacemaker return code
1468  */
1469 static int
1470 tls_client_handshake(lrmd_t *lrmd)
     /*  */
1471 {
1472     lrmd_private_t *native = lrmd->lrmd_private;
1473     int tls_rc = GNUTLS_E_SUCCESS;
1474     int rc = pcmk__tls_client_handshake(native->remote, TLS_HANDSHAKE_TIMEOUT,
1475                                         &tls_rc);
1476 
1477     if (rc != pcmk_rc_ok) {
1478         tls_handshake_failed(lrmd, tls_rc, rc);
1479     }
1480 
1481     return rc;
1482 }
1483 
1484 /*!
1485  * \internal
1486  * \brief Add trigger and file descriptor mainloop sources for TLS
1487  *
1488  * \param[in,out] lrmd          API connection with established TLS session
1489  * \param[in]     do_handshake  Whether to perform executor handshake
1490  *
1491  * \return Standard Pacemaker return code
1492  */
1493 static int
1494 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1495 {
1496     lrmd_private_t *native = lrmd->lrmd_private;
1497     int rc = pcmk_rc_ok;
1498 
1499     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1500                                    native->server, native->port);
1501 
1502     struct mainloop_fd_callbacks tls_fd_callbacks = {
1503         .dispatch = lrmd_tls_dispatch,
1504         .destroy = lrmd_tls_connection_destroy,
1505     };
1506 
1507     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1508                                                   lrmd_tls_dispatch, lrmd);
1509     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1510                                      &tls_fd_callbacks);
1511 
1512     /* Async connections lose the client name provided by the API caller, so we
1513      * have to use our generated name here to perform the executor handshake.
1514      *
1515      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1516      * that name in this function instead of generating one anyway.
1517      */
1518     if (do_handshake) {
1519         rc = lrmd_handshake(lrmd, name);
1520         rc = pcmk_legacy2rc(rc);
1521     }
1522     free(name);
1523     return rc;
1524 }
1525 
1526 struct handshake_data_s {
1527     lrmd_t *lrmd;
1528     time_t start_time;
1529     int timeout_sec;
1530 };
1531 
1532 static gboolean
1533 try_handshake_cb(gpointer user_data)
     /*  */
1534 {
1535     struct handshake_data_s *hs = user_data;
1536     lrmd_t *lrmd = hs->lrmd;
1537     lrmd_private_t *native = lrmd->lrmd_private;
1538     pcmk__remote_t *remote = native->remote;
1539 
1540     int rc = pcmk_rc_ok;
1541     int tls_rc = GNUTLS_E_SUCCESS;
1542 
1543     if (time(NULL) >= hs->start_time + hs->timeout_sec) {
1544         rc = ETIME;
1545 
1546         tls_handshake_failed(lrmd, GNUTLS_E_TIMEDOUT, rc);
1547         free(hs);
1548         return 0;
1549     }
1550 
1551     rc = pcmk__tls_client_try_handshake(remote, &tls_rc);
1552 
1553     if (rc == pcmk_rc_ok) {
1554         tls_handshake_succeeded(lrmd);
1555         free(hs);
1556         return 0;
1557     } else if (rc == EAGAIN) {
1558         mainloop_set_trigger(native->handshake_trigger);
1559         return 1;
1560     } else {
1561         rc = EKEYREJECTED;
1562         tls_handshake_failed(lrmd, tls_rc, rc);
1563         free(hs);
1564         return 0;
1565     }
1566 }
1567 
1568 static void
1569 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1570 {
1571     lrmd_t *lrmd = userdata;
1572     lrmd_private_t *native = lrmd->lrmd_private;
1573     gnutls_datum_t psk_key = { NULL, 0 };
1574     int tls_rc = GNUTLS_E_SUCCESS;
1575 
1576     native->async_timer = 0;
1577 
1578     if (rc != pcmk_rc_ok) {
1579         lrmd_tls_connection_destroy(lrmd);
1580         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1581                  CRM_XS " rc=%d",
1582                  native->server, native->port, pcmk_rc_str(rc), rc);
1583         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1584         return;
1585     }
1586 
1587     /* The TCP connection was successful, so establish the TLS connection. */
1588 
1589     native->sock = sock;
1590 
1591     rc = lrmd__init_remote_key(&psk_key);
1592     if (rc != pcmk_rc_ok) {
1593         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1594                  CRM_XS " rc=%d",
1595                  native->server, native->port, pcmk_rc_str(rc), rc);
1596         lrmd_tls_connection_destroy(lrmd);
1597         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1598         return;
1599     }
1600 
1601     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1602     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1603     gnutls_free(psk_key.data);
1604 
1605     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1606                                                         GNUTLS_CRD_PSK,
1607                                                         native->psk_cred_c);
1608     if (native->remote->tls_session == NULL) {
1609         lrmd_tls_connection_destroy(lrmd);
1610         report_async_connection_result(lrmd, -EPROTO);
1611         return;
1612     }
1613 
1614     /* If the TLS handshake immediately succeeds or fails, we can handle that
1615      * now without having to deal with mainloops and retries.  Otherwise, add a
1616      * trigger to keep trying until we get a result (or it times out).
1617      */
1618     rc = pcmk__tls_client_try_handshake(native->remote, &tls_rc);
1619     if (rc == EAGAIN) {
1620         struct handshake_data_s *hs = NULL;
1621 
1622         if (native->handshake_trigger != NULL) {
1623             return;
1624         }
1625 
1626         hs = pcmk__assert_alloc(1, sizeof(struct handshake_data_s));
1627         hs->lrmd = lrmd;
1628         hs->start_time = time(NULL);
1629         hs->timeout_sec = TLS_HANDSHAKE_TIMEOUT;
1630 
1631         native->handshake_trigger = mainloop_add_trigger(G_PRIORITY_LOW, try_handshake_cb, hs);
1632         mainloop_set_trigger(native->handshake_trigger);
1633 
1634     } else if (rc == pcmk_rc_ok) {
1635         tls_handshake_succeeded(lrmd);
1636 
1637     } else {
1638         tls_handshake_failed(lrmd, tls_rc, rc);
1639     }
1640 }
1641 
1642 static int
1643 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1644 {
1645     int rc = pcmk_rc_ok;
1646     int timer_id = 0;
1647     lrmd_private_t *native = lrmd->lrmd_private;
1648 
1649     lrmd_gnutls_global_init();
1650     native->sock = -1;
1651     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1652                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1653     if (rc != pcmk_rc_ok) {
1654         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1655                  CRM_XS " rc=%d",
1656                  native->server, native->port, pcmk_rc_str(rc), rc);
1657         return rc;
1658     }
1659     native->async_timer = timer_id;
1660     return rc;
1661 }
1662 
1663 static int
1664 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1665 {
1666     int rc = pcmk_rc_ok;
1667 
1668     lrmd_private_t *native = lrmd->lrmd_private;
1669     gnutls_datum_t psk_key = { NULL, 0 };
1670 
1671     lrmd_gnutls_global_init();
1672 
1673     native->sock = -1;
1674     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1675                               &(native->sock), NULL, NULL);
1676     if (rc != pcmk_rc_ok) {
1677         crm_warn("Pacemaker Remote connection to %s:%d failed: %s "
1678                  CRM_XS " rc=%d",
1679                  native->server, native->port, pcmk_rc_str(rc), rc);
1680         lrmd_tls_connection_destroy(lrmd);
1681         return ENOTCONN;
1682     }
1683 
1684     rc = lrmd__init_remote_key(&psk_key);
1685     if (rc != pcmk_rc_ok) {
1686         lrmd_tls_connection_destroy(lrmd);
1687         return rc;
1688     }
1689 
1690     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1691     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1692     gnutls_free(psk_key.data);
1693 
1694     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1695                                                         GNUTLS_CRD_PSK,
1696                                                         native->psk_cred_c);
1697     if (native->remote->tls_session == NULL) {
1698         lrmd_tls_connection_destroy(lrmd);
1699         return EPROTO;
1700     }
1701 
1702     if (tls_client_handshake(lrmd) != pcmk_rc_ok) {
1703         return EKEYREJECTED;
1704     }
1705 
1706     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1707              native->port);
1708 
1709     if (fd) {
1710         *fd = native->sock;
1711     } else {
1712        rc = add_tls_to_mainloop(lrmd, false);
1713     }
1714     return rc;
1715 }
1716 #endif
1717 
1718 static int
1719 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1720 {
1721     int rc = -ENOTCONN;
1722     lrmd_private_t *native = lrmd->lrmd_private;
1723 
1724     switch (native->type) {
1725         case pcmk__client_ipc:
1726             rc = lrmd_ipc_connect(lrmd, fd);
1727             rc = pcmk_rc2legacy(rc);
1728             break;
1729 #ifdef HAVE_GNUTLS_GNUTLS_H
1730         case pcmk__client_tls:
1731             rc = lrmd_tls_connect(lrmd, fd);
1732             rc = pcmk_rc2legacy(rc);
1733             break;
1734 #endif
1735         default:
1736             crm_err("Unsupported executor connection type (bug?): %d",
1737                     native->type);
1738             rc = -EPROTONOSUPPORT;
1739     }
1740 
1741     if (rc == pcmk_ok) {
1742         rc = lrmd_handshake(lrmd, name);
1743     }
1744 
1745     return rc;
1746 }
1747 
1748 static int
1749 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1750 {
1751     int rc = pcmk_ok;
1752     lrmd_private_t *native = lrmd->lrmd_private;
1753 
1754     CRM_CHECK(native && native->callback, return -EINVAL);
1755 
1756     switch (native->type) {
1757         case pcmk__client_ipc:
1758             /* fake async connection with ipc.  it should be fast
1759              * enough that we gain very little from async */
1760             rc = lrmd_api_connect(lrmd, name, NULL);
1761             if (!rc) {
1762                 report_async_connection_result(lrmd, rc);
1763             }
1764             break;
1765 #ifdef HAVE_GNUTLS_GNUTLS_H
1766         case pcmk__client_tls:
1767             rc = lrmd_tls_connect_async(lrmd, timeout);
1768             break;
1769 #endif
1770         default:
1771             crm_err("Unsupported executor connection type (bug?): %d",
1772                     native->type);
1773             rc = -EPROTONOSUPPORT;
1774     }
1775 
1776     return rc;
1777 }
1778 
1779 static void
1780 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1781 {
1782     lrmd_private_t *native = lrmd->lrmd_private;
1783 
1784     if (native->source != NULL) {
1785         /* Attached to mainloop */
1786         mainloop_del_ipc_client(native->source);
1787         native->source = NULL;
1788         native->ipc = NULL;
1789 
1790     } else if (native->ipc) {
1791         /* Not attached to mainloop */
1792         crm_ipc_t *ipc = native->ipc;
1793 
1794         native->ipc = NULL;
1795         crm_ipc_close(ipc);
1796         crm_ipc_destroy(ipc);
1797     }
1798 }
1799 
1800 #ifdef HAVE_GNUTLS_GNUTLS_H
1801 static void
1802 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1803 {
1804     lrmd_private_t *native = lrmd->lrmd_private;
1805 
1806     if (native->remote->tls_session) {
1807         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1808         gnutls_deinit(*native->remote->tls_session);
1809         gnutls_free(native->remote->tls_session);
1810         native->remote->tls_session = NULL;
1811     }
1812 
1813     if (native->async_timer) {
1814         g_source_remove(native->async_timer);
1815         native->async_timer = 0;
1816     }
1817 
1818     if (native->source != NULL) {
1819         /* Attached to mainloop */
1820         mainloop_del_ipc_client(native->source);
1821         native->source = NULL;
1822 
1823     } else if (native->sock) {
1824         close(native->sock);
1825         native->sock = 0;
1826     }
1827 
1828     if (native->pending_notify) {
1829         g_list_free_full(native->pending_notify, lrmd_free_xml);
1830         native->pending_notify = NULL;
1831     }
1832 }
1833 #endif
1834 
1835 static int
1836 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1837 {
1838     lrmd_private_t *native = lrmd->lrmd_private;
1839     int rc = pcmk_ok;
1840 
1841     switch (native->type) {
1842         case pcmk__client_ipc:
1843             crm_debug("Disconnecting from local executor");
1844             lrmd_ipc_disconnect(lrmd);
1845             break;
1846 #ifdef HAVE_GNUTLS_GNUTLS_H
1847         case pcmk__client_tls:
1848             crm_debug("Disconnecting from remote executor on %s",
1849                       native->remote_nodename);
1850             lrmd_tls_disconnect(lrmd);
1851             break;
1852 #endif
1853         default:
1854             crm_err("Unsupported executor connection type (bug?): %d",
1855                     native->type);
1856             rc = -EPROTONOSUPPORT;
1857     }
1858 
1859     free(native->token);
1860     native->token = NULL;
1861 
1862     free(native->peer_version);
1863     native->peer_version = NULL;
1864     return rc;
1865 }
1866 
1867 static int
1868 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1869                       const char *rsc_id,
1870                       const char *class,
1871                       const char *provider, const char *type, enum lrmd_call_options options)
1872 {
1873     int rc = pcmk_ok;
1874     xmlNode *data = NULL;
1875 
1876     if (!class || !type || !rsc_id) {
1877         return -EINVAL;
1878     }
1879     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1880         && (provider == NULL)) {
1881         return -EINVAL;
1882     }
1883 
1884     data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1885 
1886     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1887     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1888     crm_xml_add(data, PCMK__XA_LRMD_CLASS, class);
1889     crm_xml_add(data, PCMK__XA_LRMD_PROVIDER, provider);
1890     crm_xml_add(data, PCMK__XA_LRMD_TYPE, type);
1891     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1892     free_xml(data);
1893 
1894     return rc;
1895 }
1896 
1897 static int
1898 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1899 {
1900     int rc = pcmk_ok;
1901     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1902 
1903     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1904     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1905     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1906     free_xml(data);
1907 
1908     return rc;
1909 }
1910 
1911 lrmd_rsc_info_t *
1912 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1913                   const char *provider, const char *type)
1914 {
1915     lrmd_rsc_info_t *rsc_info = pcmk__assert_alloc(1, sizeof(lrmd_rsc_info_t));
1916 
1917     rsc_info->id = pcmk__str_copy(rsc_id);
1918     rsc_info->standard = pcmk__str_copy(standard);
1919     rsc_info->provider = pcmk__str_copy(provider);
1920     rsc_info->type = pcmk__str_copy(type);
1921     return rsc_info;
1922 }
1923 
1924 lrmd_rsc_info_t *
1925 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1926 {
1927     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1928                              rsc_info->provider, rsc_info->type);
1929 }
1930 
1931 void
1932 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1933 {
1934     if (!rsc_info) {
1935         return;
1936     }
1937     free(rsc_info->id);
1938     free(rsc_info->type);
1939     free(rsc_info->standard);
1940     free(rsc_info->provider);
1941     free(rsc_info);
1942 }
1943 
1944 static lrmd_rsc_info_t *
1945 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1946 {
1947     lrmd_rsc_info_t *rsc_info = NULL;
1948     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
1949     xmlNode *output = NULL;
1950     const char *class = NULL;
1951     const char *provider = NULL;
1952     const char *type = NULL;
1953 
1954     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
1955     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
1956     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1957     free_xml(data);
1958 
1959     if (!output) {
1960         return NULL;
1961     }
1962 
1963     class = crm_element_value(output, PCMK__XA_LRMD_CLASS);
1964     provider = crm_element_value(output, PCMK__XA_LRMD_PROVIDER);
1965     type = crm_element_value(output, PCMK__XA_LRMD_TYPE);
1966 
1967     if (!class || !type) {
1968         free_xml(output);
1969         return NULL;
1970     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1971                && !provider) {
1972         free_xml(output);
1973         return NULL;
1974     }
1975 
1976     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1977     free_xml(output);
1978     return rsc_info;
1979 }
1980 
1981 void
1982 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1983 {
1984     if (op_info) {
1985         free(op_info->rsc_id);
1986         free(op_info->action);
1987         free(op_info->interval_ms_s);
1988         free(op_info->timeout_ms_s);
1989         free(op_info);
1990     }
1991 }
1992 
1993 static int
1994 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1995                            enum lrmd_call_options options, GList **output)
1996 {
1997     xmlNode *data = NULL;
1998     xmlNode *output_xml = NULL;
1999     int rc = pcmk_ok;
2000 
2001     if (output == NULL) {
2002         return -EINVAL;
2003     }
2004     *output = NULL;
2005 
2006     // Send request
2007     if (rsc_id) {
2008         data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2009         crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2010         crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2011     }
2012     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
2013                            timeout_ms, options, TRUE);
2014     if (data) {
2015         free_xml(data);
2016     }
2017 
2018     // Process reply
2019     if ((rc != pcmk_ok) || (output_xml == NULL)) {
2020         return rc;
2021     }
2022     for (const xmlNode *rsc_xml = pcmk__xe_first_child(output_xml,
2023                                                        PCMK__XE_LRMD_RSC, NULL,
2024                                                        NULL);
2025          (rsc_xml != NULL) && (rc == pcmk_ok);
2026          rsc_xml = pcmk__xe_next_same(rsc_xml)) {
2027 
2028         rsc_id = crm_element_value(rsc_xml, PCMK__XA_LRMD_RSC_ID);
2029         if (rsc_id == NULL) {
2030             crm_err("Could not parse recurring operation information from executor");
2031             continue;
2032         }
2033         for (const xmlNode *op_xml = pcmk__xe_first_child(rsc_xml,
2034                                                           PCMK__XE_LRMD_RSC_OP,
2035                                                           NULL, NULL);
2036              op_xml != NULL; op_xml = pcmk__xe_next_same(op_xml)) {
2037 
2038             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
2039 
2040             if (op_info == NULL) {
2041                 rc = -ENOMEM;
2042                 break;
2043             }
2044             op_info->rsc_id = strdup(rsc_id);
2045             op_info->action = crm_element_value_copy(op_xml,
2046                                                      PCMK__XA_LRMD_RSC_ACTION);
2047             op_info->interval_ms_s =
2048                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_RSC_INTERVAL);
2049             op_info->timeout_ms_s =
2050                 crm_element_value_copy(op_xml, PCMK__XA_LRMD_TIMEOUT);
2051             *output = g_list_prepend(*output, op_info);
2052         }
2053     }
2054     free_xml(output_xml);
2055     return rc;
2056 }
2057 
2058 
2059 static void
2060 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
2061 {
2062     lrmd_private_t *native = lrmd->lrmd_private;
2063 
2064     native->callback = callback;
2065 }
2066 
2067 void
2068 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
2069 {
2070     lrmd_private_t *native = lrmd->lrmd_private;
2071 
2072     native->proxy_callback = callback;
2073     native->proxy_callback_userdata = userdata;
2074 }
2075 
2076 void
2077 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
2078 {
2079     lrmd_private_t *native = lrmd->lrmd_private;
2080 
2081     if (native->proxy_callback) {
2082         crm_log_xml_trace(msg, "PROXY_INBOUND");
2083         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
2084     }
2085 }
2086 
2087 int
2088 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
2089 {
2090     if (lrmd == NULL) {
2091         return -ENOTCONN;
2092     }
2093     crm_xml_add(msg, PCMK__XA_LRMD_OP, CRM_OP_IPC_FWD);
2094 
2095     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
2096     return lrmd_send_xml_no_reply(lrmd, msg);
2097 }
2098 
2099 static int
2100 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
2101 {
2102     int rc = pcmk_ok;
2103     stonith_t *stonith_api = stonith_api_new();
2104 
2105     if (stonith_api == NULL) {
2106         crm_err("Could not get fence agent meta-data: API memory allocation failed");
2107         return -ENOMEM;
2108     }
2109 
2110     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
2111                                      provider, output, 0);
2112     if ((rc == pcmk_ok) && (*output == NULL)) {
2113         rc = -EIO;
2114     }
2115     stonith_api->cmds->free(stonith_api);
2116     return rc;
2117 }
2118 
2119 static int
2120 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
2121                       const char *type, char **output,
2122                       enum lrmd_call_options options)
2123 {
2124     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
2125                                            output, options, NULL);
2126 }
2127 
2128 static int
2129 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
2130                              const char *provider, const char *type,
2131                              char **output, enum lrmd_call_options options,
2132                              lrmd_key_value_t *params)
2133 {
2134     svc_action_t *action = NULL;
2135     GHashTable *params_table = NULL;
2136 
2137     if (!standard || !type) {
2138         lrmd_key_value_freeall(params);
2139         return -EINVAL;
2140     }
2141 
2142     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2143         lrmd_key_value_freeall(params);
2144         return stonith_get_metadata(provider, type, output);
2145     }
2146 
2147     params_table = pcmk__strkey_table(free, free);
2148     for (const lrmd_key_value_t *param = params; param; param = param->next) {
2149         pcmk__insert_dup(params_table, param->key, param->value);
2150     }
2151     action = services__create_resource_action(type, standard, provider, type,
2152                                               PCMK_ACTION_META_DATA, 0,
2153                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
2154                                               params_table, 0);
2155     lrmd_key_value_freeall(params);
2156 
2157     if (action == NULL) {
2158         return -ENOMEM;
2159     }
2160     if (action->rc != PCMK_OCF_UNKNOWN) {
2161         services_action_free(action);
2162         return -EINVAL;
2163     }
2164 
2165     if (!services_action_sync(action)) {
2166         crm_err("Failed to retrieve meta-data for %s:%s:%s",
2167                 standard, provider, type);
2168         services_action_free(action);
2169         return -EIO;
2170     }
2171 
2172     if (!action->stdout_data) {
2173         crm_err("Failed to receive meta-data for %s:%s:%s",
2174                 standard, provider, type);
2175         services_action_free(action);
2176         return -EIO;
2177     }
2178 
2179     *output = strdup(action->stdout_data);
2180     services_action_free(action);
2181 
2182     return pcmk_ok;
2183 }
2184 
2185 static int
2186 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2187               const char *userdata, guint interval_ms,
2188               int timeout,      /* ms */
2189               int start_delay,  /* ms */
2190               enum lrmd_call_options options, lrmd_key_value_t * params)
2191 {
2192     int rc = pcmk_ok;
2193     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2194     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2195     lrmd_key_value_t *tmp = NULL;
2196 
2197     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2198     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2199     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2200     crm_xml_add(data, PCMK__XA_LRMD_RSC_USERDATA_STR, userdata);
2201     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2202     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2203     crm_xml_add_int(data, PCMK__XA_LRMD_RSC_START_DELAY, start_delay);
2204 
2205     for (tmp = params; tmp; tmp = tmp->next) {
2206         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2207     }
2208 
2209     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2210     free_xml(data);
2211 
2212     lrmd_key_value_freeall(params);
2213     return rc;
2214 }
2215 
2216 /* timeout is in ms */
2217 static int
2218 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2219                     int timeout, lrmd_key_value_t *params)
2220 {
2221     int rc = pcmk_ok;
2222     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_ALERT);
2223     xmlNode *args = pcmk__xe_create(data, PCMK__XE_ATTRIBUTES);
2224     lrmd_key_value_t *tmp = NULL;
2225 
2226     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2227     crm_xml_add(data, PCMK__XA_LRMD_ALERT_ID, alert_id);
2228     crm_xml_add(data, PCMK__XA_LRMD_ALERT_PATH, alert_path);
2229     crm_xml_add_int(data, PCMK__XA_LRMD_TIMEOUT, timeout);
2230 
2231     for (tmp = params; tmp; tmp = tmp->next) {
2232         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2233     }
2234 
2235     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2236                            lrmd_opt_notify_orig_only, TRUE);
2237     free_xml(data);
2238 
2239     lrmd_key_value_freeall(params);
2240     return rc;
2241 }
2242 
2243 static int
2244 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2245                 guint interval_ms)
2246 {
2247     int rc = pcmk_ok;
2248     xmlNode *data = pcmk__xe_create(NULL, PCMK__XE_LRMD_RSC);
2249 
2250     crm_xml_add(data, PCMK__XA_LRMD_ORIGIN, __func__);
2251     crm_xml_add(data, PCMK__XA_LRMD_RSC_ACTION, action);
2252     crm_xml_add(data, PCMK__XA_LRMD_RSC_ID, rsc_id);
2253     crm_xml_add_ms(data, PCMK__XA_LRMD_RSC_INTERVAL, interval_ms);
2254     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2255     free_xml(data);
2256     return rc;
2257 }
2258 
2259 static int
2260 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2261 {
2262     int rc = 0;
2263     stonith_t *stonith_api = stonith_api_new();
2264     stonith_key_value_t *stonith_resources = NULL;
2265     stonith_key_value_t *dIter = NULL;
2266 
2267     if (stonith_api == NULL) {
2268         crm_err("Could not list fence agents: API memory allocation failed");
2269         return -ENOMEM;
2270     }
2271     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2272                                    &stonith_resources, 0);
2273     stonith_api->cmds->free(stonith_api);
2274 
2275     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2276         rc++;
2277         if (resources) {
2278             *resources = lrmd_list_add(*resources, dIter->value);
2279         }
2280     }
2281 
2282     stonith_key_value_freeall(stonith_resources, 1, 0);
2283     return rc;
2284 }
2285 
2286 static int
2287 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2288                      const char *provider)
2289 {
2290     int rc = 0;
2291     int stonith_count = 0; // Initially, whether to include stonith devices
2292 
2293     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2294         stonith_count = 1;
2295 
2296     } else {
2297         GList *gIter = NULL;
2298         GList *agents = resources_list_agents(class, provider);
2299 
2300         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2301             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2302             rc++;
2303         }
2304         g_list_free_full(agents, free);
2305 
2306         if (!class) {
2307             stonith_count = 1;
2308         }
2309     }
2310 
2311     if (stonith_count) {
2312         // Now, if stonith devices are included, how many there are
2313         stonith_count = list_stonith_agents(resources);
2314         if (stonith_count > 0) {
2315             rc += stonith_count;
2316         }
2317     }
2318     if (rc == 0) {
2319         crm_notice("No agents found for class %s", class);
2320         rc = -EPROTONOSUPPORT;
2321     }
2322     return rc;
2323 }
2324 
2325 static bool
2326 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2327 {
2328     bool found = false;
2329     GList *agents = NULL;
2330     GList *gIter2 = NULL;
2331 
2332     agents = resources_list_agents(class, provider);
2333     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2334         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2335             found = true;
2336         }
2337     }
2338     g_list_free_full(agents, free);
2339     return found;
2340 }
2341 
2342 static int
2343 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2344 {
2345     int rc = pcmk_ok;
2346     char *provider = NULL;
2347     GList *ocf_providers = NULL;
2348     GList *gIter = NULL;
2349 
2350     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2351 
2352     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2353         provider = gIter->data;
2354         if (!agent || does_provider_have_agent(agent, provider,
2355                                                PCMK_RESOURCE_CLASS_OCF)) {
2356             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2357             rc++;
2358         }
2359     }
2360 
2361     g_list_free_full(ocf_providers, free);
2362     return rc;
2363 }
2364 
2365 static int
2366 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2367 {
2368     int rc = 0;
2369     GList *standards = NULL;
2370     GList *gIter = NULL;
2371 
2372     standards = resources_list_standards();
2373 
2374     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2375         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2376         rc++;
2377     }
2378 
2379     if (list_stonith_agents(NULL) > 0) {
2380         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2381         rc++;
2382     }
2383 
2384     g_list_free_full(standards, free);
2385     return rc;
2386 }
2387 
2388 /*!
2389  * \internal
2390  * \brief Create an executor API object
2391  *
2392  * \param[out] api       Will be set to newly created API object (it is the
2393  *                       caller's responsibility to free this value with
2394  *                       lrmd_api_delete() if this function succeeds)
2395  * \param[in]  nodename  If the object will be used for a remote connection,
2396  *                       the node name to use in cluster for remote executor
2397  * \param[in]  server    If the object will be used for a remote connection,
2398  *                       the resolvable host name to connect to
2399  * \param[in]  port      If the object will be used for a remote connection,
2400  *                       port number on \p server to connect to
2401  *
2402  * \return Standard Pacemaker return code
2403  * \note If the caller leaves one of \p nodename or \p server NULL, the other's
2404  *       value will be used for both. If the caller leaves both NULL, an API
2405  *       object will be created for a local executor connection.
2406  */
2407 int
2408 lrmd__new(lrmd_t **api, const char *nodename, const char *server, int port)
     /*  */
2409 {
2410     lrmd_private_t *pvt = NULL;
2411 
2412     if (api == NULL) {
2413         return EINVAL;
2414     }
2415     *api = NULL;
2416 
2417     // Allocate all memory needed
2418 
2419     *api = calloc(1, sizeof(lrmd_t));
2420     if (*api == NULL) {
2421         return ENOMEM;
2422     }
2423 
2424     pvt = calloc(1, sizeof(lrmd_private_t));
2425     if (pvt == NULL) {
2426         lrmd_api_delete(*api);
2427         *api = NULL;
2428         return ENOMEM;
2429     }
2430     (*api)->lrmd_private = pvt;
2431 
2432     // @TODO Do we need to do this for local connections?
2433     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2434 
2435     (*api)->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2436 
2437     if ((pvt->remote == NULL) || ((*api)->cmds == NULL)) {
2438         lrmd_api_delete(*api);
2439         *api = NULL;
2440         return ENOMEM;
2441     }
2442 
2443     // Set methods
2444     (*api)->cmds->connect = lrmd_api_connect;
2445     (*api)->cmds->connect_async = lrmd_api_connect_async;
2446     (*api)->cmds->is_connected = lrmd_api_is_connected;
2447     (*api)->cmds->poke_connection = lrmd_api_poke_connection;
2448     (*api)->cmds->disconnect = lrmd_api_disconnect;
2449     (*api)->cmds->register_rsc = lrmd_api_register_rsc;
2450     (*api)->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2451     (*api)->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2452     (*api)->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2453     (*api)->cmds->set_callback = lrmd_api_set_callback;
2454     (*api)->cmds->get_metadata = lrmd_api_get_metadata;
2455     (*api)->cmds->exec = lrmd_api_exec;
2456     (*api)->cmds->cancel = lrmd_api_cancel;
2457     (*api)->cmds->list_agents = lrmd_api_list_agents;
2458     (*api)->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2459     (*api)->cmds->list_standards = lrmd_api_list_standards;
2460     (*api)->cmds->exec_alert = lrmd_api_exec_alert;
2461     (*api)->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2462 
2463     if ((nodename == NULL) && (server == NULL)) {
2464         pvt->type = pcmk__client_ipc;
2465     } else {
2466 #ifdef HAVE_GNUTLS_GNUTLS_H
2467         if (nodename == NULL) {
2468             nodename = server;
2469         } else if (server == NULL) {
2470             server = nodename;
2471         }
2472         pvt->type = pcmk__client_tls;
2473         pvt->remote_nodename = strdup(nodename);
2474         pvt->server = strdup(server);
2475         if ((pvt->remote_nodename == NULL) || (pvt->server == NULL)) {
2476             lrmd_api_delete(*api);
2477             *api = NULL;
2478             return ENOMEM;
2479         }
2480         pvt->port = port;
2481         if (pvt->port == 0) {
2482             pvt->port = crm_default_remote_port();
2483         }
2484 #else
2485         crm_err("Cannot communicate with Pacemaker Remote "
2486                 "because GnuTLS is not enabled for this build");
2487         lrmd_api_delete(*api);
2488         *api = NULL;
2489         return EOPNOTSUPP;
2490 #endif
2491     }
2492     return pcmk_rc_ok;
2493 }
2494 
2495 lrmd_t *
2496 lrmd_api_new(void)
     /*  */
2497 {
2498     lrmd_t *api = NULL;
2499 
2500     pcmk__assert(lrmd__new(&api, NULL, NULL, 0) == pcmk_rc_ok);
2501     return api;
2502 }
2503 
2504 lrmd_t *
2505 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2506 {
2507     lrmd_t *api = NULL;
2508 
2509     pcmk__assert(lrmd__new(&api, nodename, server, port) == pcmk_rc_ok);
2510     return api;
2511 }
2512 
2513 void
2514 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2515 {
2516     if (lrmd == NULL) {
2517         return;
2518     }
2519     if (lrmd->cmds != NULL) { // Never NULL, but make static analysis happy
2520         if (lrmd->cmds->disconnect != NULL) { // Also never really NULL
2521             lrmd->cmds->disconnect(lrmd); // No-op if already disconnected
2522         }
2523         free(lrmd->cmds);
2524     }
2525     if (lrmd->lrmd_private != NULL) {
2526         lrmd_private_t *native = lrmd->lrmd_private;
2527 
2528 #ifdef HAVE_GNUTLS_GNUTLS_H
2529         free(native->server);
2530 #endif
2531         free(native->remote_nodename);
2532         free(native->remote);
2533         free(native->token);
2534         free(native->peer_version);
2535         free(lrmd->lrmd_private);
2536     }
2537     free(lrmd);
2538 }
2539 
2540 struct metadata_cb {
2541      void (*callback)(int pid, const pcmk__action_result_t *result,
2542                       void *user_data);
2543      void *user_data;
2544 };
2545 
2546 /*!
2547  * \internal
2548  * \brief Process asynchronous metadata completion
2549  *
2550  * \param[in,out] action  Metadata action that completed
2551  */
2552 static void
2553 metadata_complete(svc_action_t *action)
     /*  */
2554 {
2555     struct metadata_cb *metadata_cb = (struct metadata_cb *) action->cb_data;
2556     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2557 
2558     pcmk__set_result(&result, action->rc, action->status,
2559                      services__exit_reason(action));
2560     pcmk__set_result_output(&result, action->stdout_data, action->stderr_data);
2561 
2562     metadata_cb->callback(0, &result, metadata_cb->user_data);
2563     result.action_stdout = NULL; // Prevent free, because action owns it
2564     result.action_stderr = NULL; // Prevent free, because action owns it
2565     pcmk__reset_result(&result);
2566     free(metadata_cb);
2567 }
2568 
2569 /*!
2570  * \internal
2571  * \brief Retrieve agent metadata asynchronously
2572  *
2573  * \param[in]     rsc        Resource agent specification
2574  * \param[in]     callback   Function to call with result (this will always be
2575  *                           called, whether by this function directly or later
2576  *                           via the main loop, and on success the metadata will
2577  *                           be in its result argument's action_stdout)
2578  * \param[in,out] user_data  User data to pass to callback
2579  *
2580  * \return Standard Pacemaker return code
2581  * \note This function is not a lrmd_api_operations_t method because it does not
2582  *       need an lrmd_t object and does not go through the executor, but
2583  *       executes the agent directly.
2584  */
2585 int
2586 lrmd__metadata_async(const lrmd_rsc_info_t *rsc,
     /*  */
2587                      void (*callback)(int pid,
2588                                       const pcmk__action_result_t *result,
2589                                       void *user_data),
2590                      void *user_data)
2591 {
2592     svc_action_t *action = NULL;
2593     struct metadata_cb *metadata_cb = NULL;
2594     pcmk__action_result_t result = PCMK__UNKNOWN_RESULT;
2595 
2596     CRM_CHECK(callback != NULL, return EINVAL);
2597 
2598     if ((rsc == NULL) || (rsc->standard == NULL) || (rsc->type == NULL)) {
2599         pcmk__set_result(&result, PCMK_OCF_NOT_CONFIGURED,
2600                          PCMK_EXEC_ERROR_FATAL,
2601                          "Invalid resource specification");
2602         callback(0, &result, user_data);
2603         pcmk__reset_result(&result);
2604         return EINVAL;
2605     }
2606 
2607     if (strcmp(rsc->standard, PCMK_RESOURCE_CLASS_STONITH) == 0) {
2608         return stonith__metadata_async(rsc->type,
2609                                        PCMK_DEFAULT_METADATA_TIMEOUT_MS / 1000,
2610                                        callback, user_data);
2611     }
2612 
2613     action = services__create_resource_action(pcmk__s(rsc->id, rsc->type),
2614                                               rsc->standard, rsc->provider,
2615                                               rsc->type,
2616                                               PCMK_ACTION_META_DATA, 0,
2617                                               PCMK_DEFAULT_METADATA_TIMEOUT_MS,
2618                                               NULL, 0);
2619     if (action == NULL) {
2620         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2621                          "Out of memory");
2622         callback(0, &result, user_data);
2623         pcmk__reset_result(&result);
2624         return ENOMEM;
2625     }
2626     if (action->rc != PCMK_OCF_UNKNOWN) {
2627         pcmk__set_result(&result, action->rc, action->status,
2628                          services__exit_reason(action));
2629         callback(0, &result, user_data);
2630         pcmk__reset_result(&result);
2631         services_action_free(action);
2632         return EINVAL;
2633     }
2634 
2635     action->cb_data = calloc(1, sizeof(struct metadata_cb));
2636     if (action->cb_data == NULL) {
2637         services_action_free(action);
2638         pcmk__set_result(&result, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
2639                          "Out of memory");
2640         callback(0, &result, user_data);
2641         pcmk__reset_result(&result);
2642         return ENOMEM;
2643     }
2644 
2645     metadata_cb = (struct metadata_cb *) action->cb_data;
2646     metadata_cb->callback = callback;
2647     metadata_cb->user_data = user_data;
2648     if (!services_action_async(action, metadata_complete)) {
2649         services_action_free(action);
2650         return pcmk_rc_error; // @TODO Derive from action->rc and ->status
2651     }
2652 
2653     // The services library has taken responsibility for action
2654     return pcmk_rc_ok;
2655 }
2656 
2657 /*!
2658  * \internal
2659  * \brief Set the result of an executor event
2660  *
2661  * \param[in,out] event        Executor event to set
2662  * \param[in]     rc           OCF exit status of event
2663  * \param[in]     op_status    Executor status of event
2664  * \param[in]     exit_reason  Human-friendly description of event
2665  */
2666 void
2667 lrmd__set_result(lrmd_event_data_t *event, enum ocf_exitcode rc, int op_status,
     /*  */
2668                  const char *exit_reason)
2669 {
2670     if (event == NULL) {
2671         return;
2672     }
2673 
2674     event->rc = rc;
2675     event->op_status = op_status;
2676 
2677     // lrmd_event_data_t has (const char *) members that lrmd_free_event() frees
2678     pcmk__str_update((char **) &event->exit_reason, exit_reason);
2679 }
2680 
2681 /*!
2682  * \internal
2683  * \brief Clear an executor event's exit reason, output, and error output
2684  *
2685  * \param[in,out] event  Executor event to reset
2686  */
2687 void
2688 lrmd__reset_result(lrmd_event_data_t *event)
     /*  */
2689 {
2690     if (event == NULL) {
2691         return;
2692     }
2693 
2694     free((void *) event->exit_reason);
2695     event->exit_reason = NULL;
2696 
2697     free((void *) event->output);
2698     event->output = NULL;
2699 }
2700 
2701 /*!
2702  * \internal
2703  * \brief Get the uptime of a remote resource connection
2704  *
2705  * When the cluster connects to a remote resource, part of that resource's
2706  * handshake includes the uptime of the remote resource's connection.  This
2707  * uptime is stored in the lrmd_t object.
2708  *
2709  * \return The connection's uptime, or -1 if unknown
2710  */
2711 time_t
2712 lrmd__uptime(lrmd_t *lrmd)
     /*  */
2713 {
2714     lrmd_private_t *native = lrmd->lrmd_private;
2715 
2716     if (native->remote == NULL) {
2717         return -1;
2718     } else {
2719         return native->remote->uptime;
2720     }
2721 }
2722 
2723 const char *
2724 lrmd__node_start_state(lrmd_t *lrmd)
     /*  */
2725 {
2726     lrmd_private_t *native = lrmd->lrmd_private;
2727 
2728     if (native->remote == NULL) {
2729         return NULL;
2730     } else {
2731         return native->remote->start_state;
2732     }
2733 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
