lib/lrmd/lrmd_client.c

/* */
This source file includes following definitions.
lrmd_list_add
lrmd_list_freeall
lrmd_key_value_add
lrmd_key_value_freeall
lrmd_new_event
lrmd_copy_event
lrmd_free_event
lrmd_dispatch_internal
lrmd_ipc_dispatch
lrmd_free_xml
remote_executor_connected
lrmd_tls_dispatch
lrmd_poll
lrmd_dispatch
lrmd_create_op
lrmd_ipc_connection_destroy
lrmd_tls_connection_destroy
lrmd__remote_send_xml
read_remote_reply
send_remote_message
lrmd_tls_send_recv
lrmd_send_xml
lrmd_send_xml_no_reply
lrmd_api_is_connected
lrmd_send_command
lrmd_api_poke_connection
lrmd__validate_remote_settings
lrmd_handshake
lrmd_ipc_connect
copy_gnutls_datum
clear_gnutls_datum
read_gnutls_key
key_is_cached
key_cache_expired
clear_key_cache
get_cached_key
cache_key
get_remote_key
lrmd__init_remote_key
lrmd_gnutls_global_init
report_async_connection_result
lrmd__tls_client_handshake
add_tls_to_mainloop
lrmd_tcp_connect_cb
lrmd_tls_connect_async
lrmd_tls_connect
lrmd_api_connect
lrmd_api_connect_async
lrmd_ipc_disconnect
lrmd_tls_disconnect
lrmd_api_disconnect
lrmd_api_register_rsc
lrmd_api_unregister_rsc
lrmd_new_rsc_info
lrmd_copy_rsc_info
lrmd_free_rsc_info
lrmd_api_get_rsc_info
lrmd_free_op_info
lrmd_api_get_recurring_ops
lrmd_api_set_callback
lrmd_internal_set_proxy_callback
lrmd_internal_proxy_dispatch
lrmd_internal_proxy_send
stonith_get_metadata
lrmd_api_get_metadata
lrmd_api_get_metadata_params
lrmd_api_exec
lrmd_api_exec_alert
lrmd_api_cancel
list_stonith_agents
lrmd_api_list_agents
does_provider_have_agent
lrmd_api_list_ocf_providers
lrmd_api_list_standards
lrmd_api_new
lrmd_remote_api_new
lrmd_api_delete
   1 /*
   2  * Copyright 2012-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <unistd.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <stdint.h>         // uint32_t, uint64_t
  16 #include <stdarg.h>
  17 #include <string.h>
  18 #include <ctype.h>
  19 #include <errno.h>
  20 
  21 #include <sys/types.h>
  22 #include <sys/wait.h>
  23 
  24 #include <glib.h>
  25 #include <dirent.h>
  26 
  27 #include <crm/crm.h>
  28 #include <crm/lrmd.h>
  29 #include <crm/lrmd_internal.h>
  30 #include <crm/services.h>
  31 #include <crm/common/mainloop.h>
  32 #include <crm/common/ipc_internal.h>
  33 #include <crm/common/remote_internal.h>
  34 #include <crm/msg_xml.h>
  35 
  36 #include <crm/stonith-ng.h>
  37 
  38 #ifdef HAVE_GNUTLS_GNUTLS_H
  39 #  undef KEYFILE
  40 #  include <gnutls/gnutls.h>
  41 #endif
  42 
  43 #include <sys/socket.h>
  44 #include <netinet/in.h>
  45 #include <netinet/ip.h>
  46 #include <arpa/inet.h>
  47 #include <netdb.h>
  48 
  49 #define MAX_TLS_RECV_WAIT 10000
  50 
  51 CRM_TRACE_INIT_DATA(lrmd);
  52 
  53 static int lrmd_api_disconnect(lrmd_t * lrmd);
  54 static int lrmd_api_is_connected(lrmd_t * lrmd);
  55 
  56 /* IPC proxy functions */
  57 int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
  58 static void lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg);
  59 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
  60 
  61 #ifdef HAVE_GNUTLS_GNUTLS_H
  62 #  define LRMD_CLIENT_HANDSHAKE_TIMEOUT 5000    /* 5 seconds */
  63 gnutls_psk_client_credentials_t psk_cred_s;
  64 static void lrmd_tls_disconnect(lrmd_t * lrmd);
  65 static int global_remote_msg_id = 0;
  66 static void lrmd_tls_connection_destroy(gpointer userdata);
  67 #endif
  68 
  69 typedef struct lrmd_private_s {
  70     uint64_t type;
  71     char *token;
  72     mainloop_io_t *source;
  73 
  74     /* IPC parameters */
  75     crm_ipc_t *ipc;
  76 
  77     pcmk__remote_t *remote;
  78 
  79     /* Extra TLS parameters */
  80     char *remote_nodename;
  81 #ifdef HAVE_GNUTLS_GNUTLS_H
  82     char *server;
  83     int port;
  84     gnutls_psk_client_credentials_t psk_cred_c;
  85 
  86     /* while the async connection is occurring, this is the id
  87      * of the connection timeout timer. */
  88     int async_timer;
  89     int sock;
  90     /* since tls requires a round trip across the network for a
  91      * request/reply, there are times where we just want to be able
  92      * to send a request from the client and not wait around (or even care
  93      * about) what the reply is. */
  94     int expected_late_replies;
  95     GList *pending_notify;
  96     crm_trigger_t *process_notify;
  97 #endif
  98 
  99     lrmd_event_callback callback;
 100 
 101     /* Internal IPC proxy msg passing for remote guests */
 102     void (*proxy_callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg);
 103     void *proxy_callback_userdata;
 104     char *peer_version;
 105 } lrmd_private_t;
 106 
 107 static lrmd_list_t *
 108 lrmd_list_add(lrmd_list_t * head, const char *value)
     /*  */
 109 {
 110     lrmd_list_t *p, *end;
 111 
 112     p = calloc(1, sizeof(lrmd_list_t));
 113     p->val = strdup(value);
 114 
 115     end = head;
 116     while (end && end->next) {
 117         end = end->next;
 118     }
 119 
 120     if (end) {
 121         end->next = p;
 122     } else {
 123         head = p;
 124     }
 125 
 126     return head;
 127 }
 128 
 129 void
 130 lrmd_list_freeall(lrmd_list_t * head)
     /*  */
 131 {
 132     lrmd_list_t *p;
 133 
 134     while (head) {
 135         char *val = (char *)head->val;
 136 
 137         p = head->next;
 138         free(val);
 139         free(head);
 140         head = p;
 141     }
 142 }
 143 
 144 lrmd_key_value_t *
 145 lrmd_key_value_add(lrmd_key_value_t * head, const char *key, const char *value)
     /*  */
 146 {
 147     lrmd_key_value_t *p, *end;
 148 
 149     p = calloc(1, sizeof(lrmd_key_value_t));
 150     p->key = strdup(key);
 151     p->value = strdup(value);
 152 
 153     end = head;
 154     while (end && end->next) {
 155         end = end->next;
 156     }
 157 
 158     if (end) {
 159         end->next = p;
 160     } else {
 161         head = p;
 162     }
 163 
 164     return head;
 165 }
 166 
 167 void
 168 lrmd_key_value_freeall(lrmd_key_value_t * head)
     /*  */
 169 {
 170     lrmd_key_value_t *p;
 171 
 172     while (head) {
 173         p = head->next;
 174         free(head->key);
 175         free(head->value);
 176         free(head);
 177         head = p;
 178     }
 179 }
 180 
 181 /*!
 182  * Create a new lrmd_event_data_t object
 183  *
 184  * \param[in] rsc_id       ID of resource involved in event
 185  * \param[in] task         Action name
 186  * \param[in] interval_ms  Action interval
 187  *
 188  * \return Newly allocated and initialized lrmd_event_data_t
 189  * \note This functions asserts on memory errors, so the return value is
 190  *       guaranteed to be non-NULL. The caller is responsible for freeing the
 191  *       result with lrmd_free_event().
 192  */
 193 lrmd_event_data_t *
 194 lrmd_new_event(const char *rsc_id, const char *task, guint interval_ms)
     /*  */
 195 {
 196     lrmd_event_data_t *event = calloc(1, sizeof(lrmd_event_data_t));
 197 
 198     CRM_ASSERT(event != NULL);
 199     if (rsc_id != NULL) {
 200         event->rsc_id = strdup(rsc_id);
 201         CRM_ASSERT(event->rsc_id != NULL);
 202     }
 203     if (task != NULL) {
 204         event->op_type = strdup(task);
 205         CRM_ASSERT(event->op_type != NULL);
 206     }
 207     event->interval_ms = interval_ms;
 208     return event;
 209 }
 210 
 211 lrmd_event_data_t *
 212 lrmd_copy_event(lrmd_event_data_t * event)
     /*  */
 213 {
 214     lrmd_event_data_t *copy = NULL;
 215 
 216     copy = calloc(1, sizeof(lrmd_event_data_t));
 217 
 218     /* This will get all the int values.
 219      * we just have to be careful not to leave any
 220      * dangling pointers to strings. */
 221     memcpy(copy, event, sizeof(lrmd_event_data_t));
 222 
 223     copy->rsc_id = event->rsc_id ? strdup(event->rsc_id) : NULL;
 224     copy->op_type = event->op_type ? strdup(event->op_type) : NULL;
 225     copy->user_data = event->user_data ? strdup(event->user_data) : NULL;
 226     copy->output = event->output ? strdup(event->output) : NULL;
 227     copy->exit_reason = event->exit_reason ? strdup(event->exit_reason) : NULL;
 228     copy->remote_nodename = event->remote_nodename ? strdup(event->remote_nodename) : NULL;
 229     copy->params = pcmk__str_table_dup(event->params);
 230 
 231     return copy;
 232 }
 233 
 234 /*!
 235  * \brief Free an executor event
 236  *
 237  * \param[in]  Executor event object to free
 238  */
 239 void
 240 lrmd_free_event(lrmd_event_data_t *event)
     /*  */
 241 {
 242     if (event == NULL) {
 243         return;
 244     }
 245     // @TODO Why are these const char *?
 246     free((void *) event->rsc_id);
 247     free((void *) event->op_type);
 248     free((void *) event->user_data);
 249     free((void *) event->output);
 250     free((void *) event->exit_reason);
 251     free((void *) event->remote_nodename);
 252     if (event->params != NULL) {
 253         g_hash_table_destroy(event->params);
 254     }
 255     free(event);
 256 }
 257 
 258 static void
 259 lrmd_dispatch_internal(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 260 {
 261     const char *type;
 262     const char *proxy_session = crm_element_value(msg, F_LRMD_IPC_SESSION);
 263     lrmd_private_t *native = lrmd->lrmd_private;
 264     lrmd_event_data_t event = { 0, };
 265 
 266     if (proxy_session != NULL) {
 267         /* this is proxy business */
 268         lrmd_internal_proxy_dispatch(lrmd, msg);
 269         return;
 270     } else if (!native->callback) {
 271         /* no callback set */
 272         crm_trace("notify event received but client has not set callback");
 273         return;
 274     }
 275 
 276     event.remote_nodename = native->remote_nodename;
 277     type = crm_element_value(msg, F_LRMD_OPERATION);
 278     crm_element_value_int(msg, F_LRMD_CALLID, &event.call_id);
 279     event.rsc_id = crm_element_value(msg, F_LRMD_RSC_ID);
 280 
 281     if (pcmk__str_eq(type, LRMD_OP_RSC_REG, pcmk__str_none)) {
 282         event.type = lrmd_event_register;
 283     } else if (pcmk__str_eq(type, LRMD_OP_RSC_UNREG, pcmk__str_none)) {
 284         event.type = lrmd_event_unregister;
 285     } else if (pcmk__str_eq(type, LRMD_OP_RSC_EXEC, pcmk__str_none)) {
 286         time_t epoch = 0;
 287 
 288         crm_element_value_int(msg, F_LRMD_TIMEOUT, &event.timeout);
 289         crm_element_value_ms(msg, F_LRMD_RSC_INTERVAL, &event.interval_ms);
 290         crm_element_value_int(msg, F_LRMD_RSC_START_DELAY, &event.start_delay);
 291         crm_element_value_int(msg, F_LRMD_EXEC_RC, (int *)&event.rc);
 292         crm_element_value_int(msg, F_LRMD_OP_STATUS, &event.op_status);
 293         crm_element_value_int(msg, F_LRMD_RSC_DELETED, &event.rsc_deleted);
 294 
 295         crm_element_value_epoch(msg, F_LRMD_RSC_RUN_TIME, &epoch);
 296         event.t_run = (unsigned int) epoch;
 297 
 298         crm_element_value_epoch(msg, F_LRMD_RSC_RCCHANGE_TIME, &epoch);
 299         event.t_rcchange = (unsigned int) epoch;
 300 
 301         crm_element_value_int(msg, F_LRMD_RSC_EXEC_TIME, (int *)&event.exec_time);
 302         crm_element_value_int(msg, F_LRMD_RSC_QUEUE_TIME, (int *)&event.queue_time);
 303 
 304         event.op_type = crm_element_value(msg, F_LRMD_RSC_ACTION);
 305         event.user_data = crm_element_value(msg, F_LRMD_RSC_USERDATA_STR);
 306         event.output = crm_element_value(msg, F_LRMD_RSC_OUTPUT);
 307         event.exit_reason = crm_element_value(msg, F_LRMD_RSC_EXIT_REASON);
 308         event.type = lrmd_event_exec_complete;
 309 
 310         event.params = xml2list(msg);
 311     } else if (pcmk__str_eq(type, LRMD_OP_NEW_CLIENT, pcmk__str_none)) {
 312         event.type = lrmd_event_new_client;
 313     } else if (pcmk__str_eq(type, LRMD_OP_POKE, pcmk__str_none)) {
 314         event.type = lrmd_event_poke;
 315     } else {
 316         return;
 317     }
 318 
 319     crm_trace("op %s notify event received", type);
 320     native->callback(&event);
 321 
 322     if (event.params) {
 323         g_hash_table_destroy(event.params);
 324     }
 325 }
 326 
 327 // \return Always 0, to indicate that IPC mainloop source should be kept
 328 static int
 329 lrmd_ipc_dispatch(const char *buffer, ssize_t length, gpointer userdata)
     /*  */
 330 {
 331     lrmd_t *lrmd = userdata;
 332     lrmd_private_t *native = lrmd->lrmd_private;
 333 
 334     if (native->callback != NULL) {
 335         xmlNode *msg = string2xml(buffer);
 336 
 337         lrmd_dispatch_internal(lrmd, msg);
 338         free_xml(msg);
 339     }
 340     return 0;
 341 }
 342 
 343 #ifdef HAVE_GNUTLS_GNUTLS_H
 344 static void
 345 lrmd_free_xml(gpointer userdata)
     /*  */
 346 {
 347     free_xml((xmlNode *) userdata);
 348 }
 349 
 350 static bool
 351 remote_executor_connected(lrmd_t * lrmd)
     /*  */
 352 {
 353     lrmd_private_t *native = lrmd->lrmd_private;
 354 
 355     return (native->remote->tls_session != NULL);
 356 }
 357 
 358 /*!
 359  * \internal
 360  * \brief TLS dispatch function (for both trigger and file descriptor sources)
 361  *
 362  * \param[in] userdata  API connection
 363  *
 364  * \return Always return a nonnegative value, which as a file descriptor
 365  *         dispatch function means keep the mainloop source, and as a
 366  *         trigger dispatch function, 0 means remove the trigger from the
 367  *         mainloop while 1 means keep it (and job completed)
 368  */
 369 static int
 370 lrmd_tls_dispatch(gpointer userdata)
     /*  */
 371 {
 372     lrmd_t *lrmd = userdata;
 373     lrmd_private_t *native = lrmd->lrmd_private;
 374     xmlNode *xml = NULL;
 375     int rc = pcmk_rc_ok;
 376 
 377     if (!remote_executor_connected(lrmd)) {
 378         crm_trace("TLS dispatch triggered after disconnect");
 379         return 0;
 380     }
 381 
 382     crm_trace("TLS dispatch triggered");
 383 
 384     /* First check if there are any pending notifies to process that came
 385      * while we were waiting for replies earlier. */
 386     if (native->pending_notify) {
 387         GList *iter = NULL;
 388 
 389         crm_trace("Processing pending notifies");
 390         for (iter = native->pending_notify; iter; iter = iter->next) {
 391             lrmd_dispatch_internal(lrmd, iter->data);
 392         }
 393         g_list_free_full(native->pending_notify, lrmd_free_xml);
 394         native->pending_notify = NULL;
 395     }
 396 
 397     /* Next read the current buffer and see if there are any messages to handle. */
 398     switch (pcmk__remote_ready(native->remote, 0)) {
 399         case pcmk_rc_ok:
 400             rc = pcmk__read_remote_message(native->remote, -1);
 401             xml = pcmk__remote_message_xml(native->remote);
 402             break;
 403         case ETIME:
 404             // Nothing to read, check if a full message is already in buffer
 405             xml = pcmk__remote_message_xml(native->remote);
 406             break;
 407         default:
 408             rc = ENOTCONN;
 409             break;
 410     }
 411     while (xml) {
 412         const char *msg_type = crm_element_value(xml, F_LRMD_REMOTE_MSG_TYPE);
 413         if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 414             lrmd_dispatch_internal(lrmd, xml);
 415         } else if (pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 416             if (native->expected_late_replies > 0) {
 417                 native->expected_late_replies--;
 418             } else {
 419                 int reply_id = 0;
 420                 crm_element_value_int(xml, F_LRMD_CALLID, &reply_id);
 421                 /* if this happens, we want to know about it */
 422                 crm_err("Got outdated Pacemaker Remote reply %d", reply_id);
 423             }
 424         }
 425         free_xml(xml);
 426         xml = pcmk__remote_message_xml(native->remote);
 427     }
 428 
 429     if (rc == ENOTCONN) {
 430         crm_info("Lost %s executor connection while reading data",
 431                  (native->remote_nodename? native->remote_nodename : "local"));
 432         lrmd_tls_disconnect(lrmd);
 433         return 0;
 434     }
 435     return 1;
 436 }
 437 #endif
 438 
 439 /* Not used with mainloop */
 440 int
 441 lrmd_poll(lrmd_t * lrmd, int timeout)
     /*  */
 442 {
 443     lrmd_private_t *native = lrmd->lrmd_private;
 444 
 445     switch (native->type) {
 446         case pcmk__client_ipc:
 447             return crm_ipc_ready(native->ipc);
 448 
 449 #ifdef HAVE_GNUTLS_GNUTLS_H
 450         case pcmk__client_tls:
 451             if (native->pending_notify) {
 452                 return 1;
 453             } else {
 454                 int rc = pcmk__remote_ready(native->remote, 0);
 455 
 456                 switch (rc) {
 457                     case pcmk_rc_ok:
 458                         return 1;
 459                     case ETIME:
 460                         return 0;
 461                     default:
 462                         return pcmk_rc2legacy(rc);
 463                 }
 464             }
 465 #endif
 466         default:
 467             crm_err("Unsupported executor connection type (bug?): %d",
 468                     native->type);
 469             return -EPROTONOSUPPORT;
 470     }
 471 }
 472 
 473 /* Not used with mainloop */
 474 bool
 475 lrmd_dispatch(lrmd_t * lrmd)
     /*  */
 476 {
 477     lrmd_private_t *private = NULL;
 478 
 479     CRM_ASSERT(lrmd != NULL);
 480 
 481     private = lrmd->lrmd_private;
 482     switch (private->type) {
 483         case pcmk__client_ipc:
 484             while (crm_ipc_ready(private->ipc)) {
 485                 if (crm_ipc_read(private->ipc) > 0) {
 486                     const char *msg = crm_ipc_buffer(private->ipc);
 487 
 488                     lrmd_ipc_dispatch(msg, strlen(msg), lrmd);
 489                 }
 490             }
 491             break;
 492 #ifdef HAVE_GNUTLS_GNUTLS_H
 493         case pcmk__client_tls:
 494             lrmd_tls_dispatch(lrmd);
 495             break;
 496 #endif
 497         default:
 498             crm_err("Unsupported executor connection type (bug?): %d",
 499                     private->type);
 500     }
 501 
 502     if (lrmd_api_is_connected(lrmd) == FALSE) {
 503         crm_err("Connection closed");
 504         return FALSE;
 505     }
 506 
 507     return TRUE;
 508 }
 509 
 510 static xmlNode *
 511 lrmd_create_op(const char *token, const char *op, xmlNode *data, int timeout,
     /*  */
 512                enum lrmd_call_options options)
 513 {
 514     xmlNode *op_msg = create_xml_node(NULL, "lrmd_command");
 515 
 516     CRM_CHECK(op_msg != NULL, return NULL);
 517     CRM_CHECK(token != NULL, return NULL);
 518 
 519     crm_xml_add(op_msg, F_XML_TAGNAME, "lrmd_command");
 520     crm_xml_add(op_msg, F_TYPE, T_LRMD);
 521     crm_xml_add(op_msg, F_LRMD_CALLBACK_TOKEN, token);
 522     crm_xml_add(op_msg, F_LRMD_OPERATION, op);
 523     crm_xml_add_int(op_msg, F_LRMD_TIMEOUT, timeout);
 524     crm_xml_add_int(op_msg, F_LRMD_CALLOPTS, options);
 525 
 526     if (data != NULL) {
 527         add_message_xml(op_msg, F_LRMD_CALLDATA, data);
 528     }
 529 
 530     crm_trace("Created executor %s command with call options %.8lx (%d)",
 531               op, (long)options, options);
 532     return op_msg;
 533 }
 534 
 535 static void
 536 lrmd_ipc_connection_destroy(gpointer userdata)
     /*  */
 537 {
 538     lrmd_t *lrmd = userdata;
 539     lrmd_private_t *native = lrmd->lrmd_private;
 540 
 541     crm_info("IPC connection destroyed");
 542 
 543     /* Prevent these from being cleaned up in lrmd_api_disconnect() */
 544     native->ipc = NULL;
 545     native->source = NULL;
 546 
 547     if (native->callback) {
 548         lrmd_event_data_t event = { 0, };
 549         event.type = lrmd_event_disconnect;
 550         event.remote_nodename = native->remote_nodename;
 551         native->callback(&event);
 552     }
 553 }
 554 
 555 #ifdef HAVE_GNUTLS_GNUTLS_H
 556 static void
 557 lrmd_tls_connection_destroy(gpointer userdata)
     /*  */
 558 {
 559     lrmd_t *lrmd = userdata;
 560     lrmd_private_t *native = lrmd->lrmd_private;
 561 
 562     crm_info("TLS connection destroyed");
 563 
 564     if (native->remote->tls_session) {
 565         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
 566         gnutls_deinit(*native->remote->tls_session);
 567         gnutls_free(native->remote->tls_session);
 568     }
 569     if (native->psk_cred_c) {
 570         gnutls_psk_free_client_credentials(native->psk_cred_c);
 571     }
 572     if (native->sock) {
 573         close(native->sock);
 574     }
 575     if (native->process_notify) {
 576         mainloop_destroy_trigger(native->process_notify);
 577         native->process_notify = NULL;
 578     }
 579     if (native->pending_notify) {
 580         g_list_free_full(native->pending_notify, lrmd_free_xml);
 581         native->pending_notify = NULL;
 582     }
 583 
 584     free(native->remote->buffer);
 585     native->remote->buffer = NULL;
 586     native->source = 0;
 587     native->sock = 0;
 588     native->psk_cred_c = NULL;
 589     native->remote->tls_session = NULL;
 590     native->sock = 0;
 591 
 592     if (native->callback) {
 593         lrmd_event_data_t event = { 0, };
 594         event.remote_nodename = native->remote_nodename;
 595         event.type = lrmd_event_disconnect;
 596         native->callback(&event);
 597     }
 598     return;
 599 }
 600 
 601 // \return Standard Pacemaker return code
 602 int
 603 lrmd__remote_send_xml(pcmk__remote_t *session, xmlNode *msg, uint32_t id,
     /*  */
 604                       const char *msg_type)
 605 {
 606     crm_xml_add_int(msg, F_LRMD_REMOTE_MSG_ID, id);
 607     crm_xml_add(msg, F_LRMD_REMOTE_MSG_TYPE, msg_type);
 608     return pcmk__remote_send_xml(session, msg);
 609 }
 610 
 611 // \return Standard Pacemaker return code
 612 static int
 613 read_remote_reply(lrmd_t *lrmd, int total_timeout, int expected_reply_id,
     /*  */
 614                   xmlNode **reply)
 615 {
 616     lrmd_private_t *native = lrmd->lrmd_private;
 617     time_t start = time(NULL);
 618     const char *msg_type = NULL;
 619     int reply_id = 0;
 620     int remaining_timeout = 0;
 621     int rc = pcmk_rc_ok;
 622 
 623     /* A timeout of 0 here makes no sense.  We have to wait a period of time
 624      * for the response to come back.  If -1 or 0, default to 10 seconds. */
 625     if (total_timeout <= 0 || total_timeout > MAX_TLS_RECV_WAIT) {
 626         total_timeout = MAX_TLS_RECV_WAIT;
 627     }
 628 
 629     for (*reply = NULL; *reply == NULL; ) {
 630 
 631         *reply = pcmk__remote_message_xml(native->remote);
 632         if (*reply == NULL) {
 633             /* read some more off the tls buffer if we still have time left. */
 634             if (remaining_timeout) {
 635                 remaining_timeout = total_timeout - ((time(NULL) - start) * 1000);
 636             } else {
 637                 remaining_timeout = total_timeout;
 638             }
 639             if (remaining_timeout <= 0) {
 640                 return ETIME;
 641             }
 642 
 643             rc = pcmk__read_remote_message(native->remote, remaining_timeout);
 644             if (rc != pcmk_rc_ok) {
 645                 return rc;
 646             }
 647 
 648             *reply = pcmk__remote_message_xml(native->remote);
 649             if (*reply == NULL) {
 650                 return ENOMSG;
 651             }
 652         }
 653 
 654         crm_element_value_int(*reply, F_LRMD_REMOTE_MSG_ID, &reply_id);
 655         msg_type = crm_element_value(*reply, F_LRMD_REMOTE_MSG_TYPE);
 656 
 657         if (!msg_type) {
 658             crm_err("Empty msg type received while waiting for reply");
 659             free_xml(*reply);
 660             *reply = NULL;
 661         } else if (pcmk__str_eq(msg_type, "notify", pcmk__str_casei)) {
 662             /* got a notify while waiting for reply, trigger the notify to be processed later */
 663             crm_info("queueing notify");
 664             native->pending_notify = g_list_append(native->pending_notify, *reply);
 665             if (native->process_notify) {
 666                 crm_info("notify trigger set.");
 667                 mainloop_set_trigger(native->process_notify);
 668             }
 669             *reply = NULL;
 670         } else if (!pcmk__str_eq(msg_type, "reply", pcmk__str_casei)) {
 671             /* msg isn't a reply, make some noise */
 672             crm_err("Expected a reply, got %s", msg_type);
 673             free_xml(*reply);
 674             *reply = NULL;
 675         } else if (reply_id != expected_reply_id) {
 676             if (native->expected_late_replies > 0) {
 677                 native->expected_late_replies--;
 678             } else {
 679                 crm_err("Got outdated reply, expected id %d got id %d", expected_reply_id, reply_id);
 680             }
 681             free_xml(*reply);
 682             *reply = NULL;
 683         }
 684     }
 685 
 686     if (native->remote->buffer && native->process_notify) {
 687         mainloop_set_trigger(native->process_notify);
 688     }
 689 
 690     return rc;
 691 }
 692 
 693 // \return Standard Pacemaker return code
 694 static int
 695 send_remote_message(lrmd_t *lrmd, xmlNode *msg)
     /*  */
 696 {
 697     int rc = pcmk_rc_ok;
 698     lrmd_private_t *native = lrmd->lrmd_private;
 699 
 700     global_remote_msg_id++;
 701     if (global_remote_msg_id <= 0) {
 702         global_remote_msg_id = 1;
 703     }
 704 
 705     rc = lrmd__remote_send_xml(native->remote, msg, global_remote_msg_id,
 706                                "request");
 707     if (rc != pcmk_rc_ok) {
 708         crm_err("Disconnecting because TLS message could not be sent to "
 709                 "Pacemaker Remote: %s", pcmk_rc_str(rc));
 710         lrmd_tls_disconnect(lrmd);
 711     }
 712     return rc;
 713 }
 714 
 715 static int
 716 lrmd_tls_send_recv(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 717 {
 718     int rc = 0;
 719     xmlNode *xml = NULL;
 720 
 721     if (!remote_executor_connected(lrmd)) {
 722         return -ENOTCONN;
 723     }
 724 
 725     rc = send_remote_message(lrmd, msg);
 726     if (rc != pcmk_rc_ok) {
 727         return pcmk_rc2legacy(rc);
 728     }
 729 
 730     rc = read_remote_reply(lrmd, timeout, global_remote_msg_id, &xml);
 731     if (rc != pcmk_rc_ok) {
 732         crm_err("Disconnecting remote after request %d reply not received: %s "
 733                 CRM_XS " rc=%d timeout=%dms",
 734                 global_remote_msg_id, pcmk_rc_str(rc), rc, timeout);
 735         lrmd_tls_disconnect(lrmd);
 736     }
 737 
 738     if (reply) {
 739         *reply = xml;
 740     } else {
 741         free_xml(xml);
 742     }
 743 
 744     return pcmk_rc2legacy(rc);
 745 }
 746 #endif
 747 
 748 static int
 749 lrmd_send_xml(lrmd_t * lrmd, xmlNode * msg, int timeout, xmlNode ** reply)
     /*  */
 750 {
 751     int rc = pcmk_ok;
 752     lrmd_private_t *native = lrmd->lrmd_private;
 753 
 754     switch (native->type) {
 755         case pcmk__client_ipc:
 756             rc = crm_ipc_send(native->ipc, msg, crm_ipc_client_response, timeout, reply);
 757             break;
 758 #ifdef HAVE_GNUTLS_GNUTLS_H
 759         case pcmk__client_tls:
 760             rc = lrmd_tls_send_recv(lrmd, msg, timeout, reply);
 761             break;
 762 #endif
 763         default:
 764             crm_err("Unsupported executor connection type (bug?): %d",
 765                     native->type);
 766             rc = -EPROTONOSUPPORT;
 767     }
 768 
 769     return rc;
 770 }
 771 
 772 static int
 773 lrmd_send_xml_no_reply(lrmd_t * lrmd, xmlNode * msg)
     /*  */
 774 {
 775     int rc = pcmk_ok;
 776     lrmd_private_t *native = lrmd->lrmd_private;
 777 
 778     switch (native->type) {
 779         case pcmk__client_ipc:
 780             rc = crm_ipc_send(native->ipc, msg, crm_ipc_flags_none, 0, NULL);
 781             break;
 782 #ifdef HAVE_GNUTLS_GNUTLS_H
 783         case pcmk__client_tls:
 784             rc = send_remote_message(lrmd, msg);
 785             if (rc == pcmk_rc_ok) {
 786                 /* we don't want to wait around for the reply, but
 787                  * since the request/reply protocol needs to behave the same
 788                  * as libqb, a reply will eventually come later anyway. */
 789                 native->expected_late_replies++;
 790             }
 791             rc = pcmk_rc2legacy(rc);
 792             break;
 793 #endif
 794         default:
 795             crm_err("Unsupported executor connection type (bug?): %d",
 796                     native->type);
 797             rc = -EPROTONOSUPPORT;
 798     }
 799 
 800     return rc;
 801 }
 802 
 803 static int
 804 lrmd_api_is_connected(lrmd_t * lrmd)
     /*  */
 805 {
 806     lrmd_private_t *native = lrmd->lrmd_private;
 807 
 808     switch (native->type) {
 809         case pcmk__client_ipc:
 810             return crm_ipc_connected(native->ipc);
 811 #ifdef HAVE_GNUTLS_GNUTLS_H
 812         case pcmk__client_tls:
 813             return remote_executor_connected(lrmd);
 814 #endif
 815         default:
 816             crm_err("Unsupported executor connection type (bug?): %d",
 817                     native->type);
 818             return 0;
 819     }
 820 }
 821 
 822 /*!
 823  * \internal
 824  * \brief Send a prepared API command to the executor
 825  *
 826  * \param[in]  lrmd          Existing connection to the executor
 827  * \param[in]  op            Name of API command to send
 828  * \param[in]  data          Command data XML to add to the sent command
 829  * \param[out] output_data   If expecting a reply, it will be stored here
 830  * \param[in]  timeout       Timeout in milliseconds (if 0, defaults to
 831  *                           a sensible value per the type of connection,
 832  *                           standard vs. pacemaker remote);
 833  *                           also propagated to the command XML
 834  * \param[in]  call_options  Call options to pass to server when sending
 835  * \param[in]  expect_reply  If TRUE, wait for a reply from the server;
 836  *                           must be TRUE for IPC (as opposed to TLS) clients
 837  *
 838  * \return pcmk_ok on success, -errno on error
 839  */
 840 static int
 841 lrmd_send_command(lrmd_t *lrmd, const char *op, xmlNode *data,
     /*  */
 842                   xmlNode **output_data, int timeout,
 843                   enum lrmd_call_options options, gboolean expect_reply)
 844 {
 845     int rc = pcmk_ok;
 846     lrmd_private_t *native = lrmd->lrmd_private;
 847     xmlNode *op_msg = NULL;
 848     xmlNode *op_reply = NULL;
 849 
 850     if (!lrmd_api_is_connected(lrmd)) {
 851         return -ENOTCONN;
 852     }
 853 
 854     if (op == NULL) {
 855         crm_err("No operation specified");
 856         return -EINVAL;
 857     }
 858 
 859     CRM_CHECK(native->token != NULL,;
 860         );
 861     crm_trace("Sending %s op to executor", op);
 862 
 863     op_msg = lrmd_create_op(native->token, op, data, timeout, options);
 864 
 865     if (op_msg == NULL) {
 866         return -EINVAL;
 867     }
 868 
 869     if (expect_reply) {
 870         rc = lrmd_send_xml(lrmd, op_msg, timeout, &op_reply);
 871     } else {
 872         rc = lrmd_send_xml_no_reply(lrmd, op_msg);
 873         goto done;
 874     }
 875 
 876     if (rc < 0) {
 877         crm_perror(LOG_ERR, "Couldn't perform %s operation (timeout=%d): %d", op, timeout, rc);
 878         rc = -ECOMM;
 879         goto done;
 880 
 881     } else if(op_reply == NULL) {
 882         rc = -ENOMSG;
 883         goto done;
 884     }
 885 
 886     rc = pcmk_ok;
 887     crm_trace("%s op reply received", op);
 888     if (crm_element_value_int(op_reply, F_LRMD_RC, &rc) != 0) {
 889         rc = -ENOMSG;
 890         goto done;
 891     }
 892 
 893     crm_log_xml_trace(op_reply, "Reply");
 894 
 895     if (output_data) {
 896         *output_data = op_reply;
 897         op_reply = NULL;        /* Prevent subsequent free */
 898     }
 899 
 900   done:
 901     if (lrmd_api_is_connected(lrmd) == FALSE) {
 902         crm_err("Executor disconnected");
 903     }
 904 
 905     free_xml(op_msg);
 906     free_xml(op_reply);
 907     return rc;
 908 }
 909 
 910 static int
 911 lrmd_api_poke_connection(lrmd_t * lrmd)
     /*  */
 912 {
 913     int rc;
 914     lrmd_private_t *native = lrmd->lrmd_private;
 915     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
 916 
 917     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 918     rc = lrmd_send_command(lrmd, LRMD_OP_POKE, data, NULL, 0, 0,
 919                            (native->type == pcmk__client_ipc));
 920     free_xml(data);
 921 
 922     return rc < 0 ? rc : pcmk_ok;
 923 }
 924 
 925 // \return Standard Pacemaker return code
 926 int
 927 lrmd__validate_remote_settings(lrmd_t *lrmd, GHashTable *hash)
     /*  */
 928 {
 929     int rc = pcmk_rc_ok;
 930     const char *value;
 931     lrmd_private_t *native = lrmd->lrmd_private;
 932     xmlNode *data = create_xml_node(NULL, F_LRMD_OPERATION);
 933 
 934     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
 935 
 936     value = g_hash_table_lookup(hash, "stonith-watchdog-timeout");
 937     crm_xml_add(data, F_LRMD_WATCHDOG, value);
 938 
 939     rc = lrmd_send_command(lrmd, LRMD_OP_CHECK, data, NULL, 0, 0,
 940                            (native->type == pcmk__client_ipc));
 941     free_xml(data);
 942     return (rc < 0)? pcmk_legacy2rc(rc) : pcmk_rc_ok;
 943 }
 944 
 945 static int
 946 lrmd_handshake(lrmd_t * lrmd, const char *name)
     /*  */
 947 {
 948     int rc = pcmk_ok;
 949     lrmd_private_t *native = lrmd->lrmd_private;
 950     xmlNode *reply = NULL;
 951     xmlNode *hello = create_xml_node(NULL, "lrmd_command");
 952 
 953     crm_xml_add(hello, F_TYPE, T_LRMD);
 954     crm_xml_add(hello, F_LRMD_OPERATION, CRM_OP_REGISTER);
 955     crm_xml_add(hello, F_LRMD_CLIENTNAME, name);
 956     crm_xml_add(hello, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
 957 
 958     /* advertise that we are a proxy provider */
 959     if (native->proxy_callback) {
 960         crm_xml_add(hello, F_LRMD_IS_IPC_PROVIDER, "true");
 961     }
 962 
 963     rc = lrmd_send_xml(lrmd, hello, -1, &reply);
 964 
 965     if (rc < 0) {
 966         crm_perror(LOG_DEBUG, "Couldn't complete registration with the executor API: %d", rc);
 967         rc = -ECOMM;
 968     } else if (reply == NULL) {
 969         crm_err("Did not receive registration reply");
 970         rc = -EPROTO;
 971     } else {
 972         const char *version = crm_element_value(reply, F_LRMD_PROTOCOL_VERSION);
 973         const char *msg_type = crm_element_value(reply, F_LRMD_OPERATION);
 974         const char *tmp_ticket = crm_element_value(reply, F_LRMD_CLIENTID);
 975 
 976         crm_element_value_int(reply, F_LRMD_RC, &rc);
 977 
 978         if (rc == -EPROTO) {
 979             crm_err("Executor protocol version mismatch between client (%s) and server (%s)",
 980                 LRMD_PROTOCOL_VERSION, version);
 981             crm_log_xml_err(reply, "Protocol Error");
 982 
 983         } else if (!pcmk__str_eq(msg_type, CRM_OP_REGISTER, pcmk__str_casei)) {
 984             crm_err("Invalid registration message: %s", msg_type);
 985             crm_log_xml_err(reply, "Bad reply");
 986             rc = -EPROTO;
 987         } else if (tmp_ticket == NULL) {
 988             crm_err("No registration token provided");
 989             crm_log_xml_err(reply, "Bad reply");
 990             rc = -EPROTO;
 991         } else {
 992             crm_trace("Obtained registration token: %s", tmp_ticket);
 993             native->token = strdup(tmp_ticket);
 994             native->peer_version = strdup(version?version:"1.0"); /* Included since 1.1 */
 995             rc = pcmk_ok;
 996         }
 997     }
 998 
 999     free_xml(reply);
1000     free_xml(hello);
1001 
1002     if (rc != pcmk_ok) {
1003         lrmd_api_disconnect(lrmd);
1004     }
1005     return rc;
1006 }
1007 
1008 static int
1009 lrmd_ipc_connect(lrmd_t * lrmd, int *fd)
     /*  */
1010 {
1011     int rc = pcmk_ok;
1012     lrmd_private_t *native = lrmd->lrmd_private;
1013 
1014     struct ipc_client_callbacks lrmd_callbacks = {
1015         .dispatch = lrmd_ipc_dispatch,
1016         .destroy = lrmd_ipc_connection_destroy
1017     };
1018 
1019     crm_info("Connecting to executor");
1020 
1021     if (fd) {
1022         /* No mainloop */
1023         native->ipc = crm_ipc_new(CRM_SYSTEM_LRMD, 0);
1024         if (native->ipc && crm_ipc_connect(native->ipc)) {
1025             *fd = crm_ipc_get_fd(native->ipc);
1026         } else if (native->ipc) {
1027             crm_perror(LOG_ERR, "Connection to executor failed");
1028             rc = -ENOTCONN;
1029         }
1030     } else {
1031         native->source = mainloop_add_ipc_client(CRM_SYSTEM_LRMD, G_PRIORITY_HIGH, 0, lrmd, &lrmd_callbacks);
1032         native->ipc = mainloop_get_ipc_client(native->source);
1033     }
1034 
1035     if (native->ipc == NULL) {
1036         crm_debug("Could not connect to the executor API");
1037         rc = -ENOTCONN;
1038     }
1039 
1040     return rc;
1041 }
1042 
1043 #ifdef HAVE_GNUTLS_GNUTLS_H
1044 static void
1045 copy_gnutls_datum(gnutls_datum_t *dest, gnutls_datum_t *source)
     /*  */
1046 {
1047     dest->data = gnutls_malloc(source->size);
1048     CRM_ASSERT(dest->data);
1049     memcpy(dest->data, source->data, source->size);
1050     dest->size = source->size;
1051 }
1052 
1053 static void
1054 clear_gnutls_datum(gnutls_datum_t *datum)
     /*  */
1055 {
1056     gnutls_free(datum->data);
1057     datum->data = NULL;
1058     datum->size = 0;
1059 }
1060 
1061 #define KEY_READ_LEN 256    // Chunk size for reading key from file
1062 
1063 // \return Standard Pacemaker return code
1064 static int
1065 read_gnutls_key(const char *location, gnutls_datum_t *key)
     /*  */
1066 {
1067     FILE *stream = NULL;
1068     size_t buf_len = KEY_READ_LEN;
1069 
1070     if ((location == NULL) || (key == NULL)) {
1071         return EINVAL;
1072     }
1073 
1074     stream = fopen(location, "r");
1075     if (stream == NULL) {
1076         return errno;
1077     }
1078 
1079     key->data = gnutls_malloc(buf_len);
1080     key->size = 0;
1081     while (!feof(stream)) {
1082         int next = fgetc(stream);
1083 
1084         if (next == EOF) {
1085             if (!feof(stream)) {
1086                 crm_warn("Pacemaker Remote key read was partially successful "
1087                          "(copy in memory may be corrupted)");
1088             }
1089             break;
1090         }
1091         if (key->size == buf_len) {
1092             buf_len = key->size + KEY_READ_LEN;
1093             key->data = gnutls_realloc(key->data, buf_len);
1094             CRM_ASSERT(key->data);
1095         }
1096         key->data[key->size++] = (unsigned char) next;
1097     }
1098     fclose(stream);
1099 
1100     if (key->size == 0) {
1101         clear_gnutls_datum(key);
1102         return ENOKEY;
1103     }
1104     return pcmk_rc_ok;
1105 }
1106 
1107 // Cache the most recently used Pacemaker Remote authentication key
1108 
1109 struct key_cache_s {
1110     time_t updated;         // When cached key was read (valid for 1 minute)
1111     const char *location;   // Where cached key was read from
1112     gnutls_datum_t key;     // Cached key
1113 };
1114 
1115 static bool
1116 key_is_cached(struct key_cache_s *key_cache)
     /*  */
1117 {
1118     return key_cache->updated != 0;
1119 }
1120 
1121 static bool
1122 key_cache_expired(struct key_cache_s *key_cache)
     /*  */
1123 {
1124     return (time(NULL) - key_cache->updated) >= 60;
1125 }
1126 
1127 static void
1128 clear_key_cache(struct key_cache_s *key_cache)
     /*  */
1129 {
1130     clear_gnutls_datum(&(key_cache->key));
1131     if ((key_cache->updated != 0) || (key_cache->location != NULL)) {
1132         key_cache->updated = 0;
1133         key_cache->location = NULL;
1134         crm_debug("Cleared Pacemaker Remote key cache");
1135     }
1136 }
1137 
1138 static void
1139 get_cached_key(struct key_cache_s *key_cache, gnutls_datum_t *key)
     /*  */
1140 {
1141     copy_gnutls_datum(key, &(key_cache->key));
1142     crm_debug("Using cached Pacemaker Remote key from %s",
1143               crm_str(key_cache->location));
1144 }
1145 
1146 static void
1147 cache_key(struct key_cache_s *key_cache, gnutls_datum_t *key,
     /*  */
1148           const char *location)
1149 {
1150     key_cache->updated = time(NULL);
1151     key_cache->location = location;
1152     copy_gnutls_datum(&(key_cache->key), key);
1153     crm_debug("Using (and cacheing) Pacemaker Remote key from %s",
1154               crm_str(location));
1155 }
1156 
1157 /*!
1158  * \internal
1159  * \brief Get Pacemaker Remote authentication key from file or cache
1160  *
1161  * \param[in]  location         Path to key file to try (this memory must
1162  *                              persist across all calls of this function)
1163  * \param[out] key              Key from location or cache
1164  *
1165  * \return Standard Pacemaker return code
1166  */
1167 static int
1168 get_remote_key(const char *location, gnutls_datum_t *key)
     /*  */
1169 {
1170     static struct key_cache_s key_cache = { 0, };
1171     int rc = pcmk_rc_ok;
1172 
1173     if ((location == NULL) || (key == NULL)) {
1174         return EINVAL;
1175     }
1176 
1177     if (key_is_cached(&key_cache)) {
1178         if (key_cache_expired(&key_cache)) {
1179             clear_key_cache(&key_cache);
1180         } else {
1181             get_cached_key(&key_cache, key);
1182             return pcmk_rc_ok;
1183         }
1184     }
1185 
1186     rc = read_gnutls_key(location, key);
1187     if (rc != pcmk_rc_ok) {
1188         return rc;
1189     }
1190     cache_key(&key_cache, key, location);
1191     return pcmk_rc_ok;
1192 }
1193 
1194 /*!
1195  * \internal
1196  * \brief Initialize the Pacemaker Remote authentication key
1197  *
1198  * Try loading the Pacemaker Remote authentication key from cache if available,
1199  * otherwise from these locations, in order of preference: the value of the
1200  * PCMK_authkey_location environment variable, if set; the Pacemaker default key
1201  * file location; or (for historical reasons) /etc/corosync/authkey.
1202  *
1203  * \param[out] key  Where to store key
1204  *
1205  * \return Standard Pacemaker return code
1206  */
1207 int
1208 lrmd__init_remote_key(gnutls_datum_t *key)
     /*  */
1209 {
1210     static const char *env_location = NULL;
1211     static bool need_env = true;
1212 
1213     int env_rc = pcmk_rc_ok;
1214     int default_rc = pcmk_rc_ok;
1215     int alt_rc = pcmk_rc_ok;
1216 
1217     bool env_is_default = false;
1218     bool env_is_fallback = false;
1219 
1220     if (need_env) {
1221         env_location = getenv("PCMK_authkey_location");
1222         need_env = false;
1223     }
1224 
1225     // Try location in environment variable, if set
1226     if (env_location != NULL) {
1227         env_rc = get_remote_key(env_location, key);
1228         if (env_rc == pcmk_rc_ok) {
1229             return pcmk_rc_ok;
1230         }
1231 
1232         env_is_default = !strcmp(env_location, DEFAULT_REMOTE_KEY_LOCATION);
1233         env_is_fallback = !strcmp(env_location, ALT_REMOTE_KEY_LOCATION);
1234 
1235         /* @TODO It would be more secure to fail, rather than fall back to the
1236          * default, if an explicitly set key location is not readable, and it
1237          * would be better to never use the Corosync location as a fallback.
1238          * However, that would break any deployments currently working with the
1239          * fallbacks.
1240          */
1241     }
1242 
1243     // Try default location, if environment wasn't explicitly set to it
1244     if (env_is_default) {
1245         default_rc = env_rc;
1246     } else {
1247         default_rc = get_remote_key(DEFAULT_REMOTE_KEY_LOCATION, key);
1248     }
1249 
1250     // Try fallback location, if environment wasn't set to it and default failed
1251     if (env_is_fallback) {
1252         alt_rc = env_rc;
1253     } else if (default_rc != pcmk_rc_ok) {
1254         alt_rc = get_remote_key(ALT_REMOTE_KEY_LOCATION, key);
1255     }
1256 
1257     // We have all results, so log and return
1258 
1259     if ((env_rc != pcmk_rc_ok) && (default_rc != pcmk_rc_ok)
1260         && (alt_rc != pcmk_rc_ok)) { // Environment set, everything failed
1261 
1262         crm_warn("Could not read Pacemaker Remote key from %s (%s%s%s%s%s): %s",
1263                  env_location,
1264                  env_is_default? "" : "or default location ",
1265                  env_is_default? "" : DEFAULT_REMOTE_KEY_LOCATION,
1266                  !env_is_default && !env_is_fallback? " " : "",
1267                  env_is_fallback? "" : "or fallback location ",
1268                  env_is_fallback? "" : ALT_REMOTE_KEY_LOCATION,
1269                  pcmk_rc_str(env_rc));
1270         return ENOKEY;
1271     }
1272 
1273     if (env_rc != pcmk_rc_ok) { // Environment set but failed, using a default
1274         crm_warn("Could not read Pacemaker Remote key from %s "
1275                  "(using %s location %s instead): %s",
1276                  env_location,
1277                  (default_rc == pcmk_rc_ok)? "default" : "fallback",
1278                  (default_rc == pcmk_rc_ok)? DEFAULT_REMOTE_KEY_LOCATION : ALT_REMOTE_KEY_LOCATION,
1279                  pcmk_rc_str(env_rc));
1280         return pcmk_rc_ok;
1281     }
1282 
1283     if ((default_rc != pcmk_rc_ok) && (alt_rc != pcmk_rc_ok)) {
1284         // Environment unset, defaults failed
1285         crm_warn("Could not read Pacemaker Remote key from default location %s"
1286                  " (or fallback location %s): %s",
1287                  DEFAULT_REMOTE_KEY_LOCATION, ALT_REMOTE_KEY_LOCATION,
1288                  pcmk_rc_str(default_rc));
1289         return ENOKEY;
1290     }
1291 
1292     return pcmk_rc_ok; // Environment variable unset, a default worked
1293 }
1294 
1295 static void
1296 lrmd_gnutls_global_init(void)
     /*  */
1297 {
1298     static int gnutls_init = 0;
1299 
1300     if (!gnutls_init) {
1301         crm_gnutls_global_init();
1302     }
1303     gnutls_init = 1;
1304 }
1305 #endif
1306 
1307 static void
1308 report_async_connection_result(lrmd_t * lrmd, int rc)
     /*  */
1309 {
1310     lrmd_private_t *native = lrmd->lrmd_private;
1311 
1312     if (native->callback) {
1313         lrmd_event_data_t event = { 0, };
1314         event.type = lrmd_event_connect;
1315         event.remote_nodename = native->remote_nodename;
1316         event.connection_rc = rc;
1317         native->callback(&event);
1318     }
1319 }
1320 
1321 #ifdef HAVE_GNUTLS_GNUTLS_H
1322 static inline int
1323 lrmd__tls_client_handshake(pcmk__remote_t *remote)
     /*  */
1324 {
1325     return pcmk__tls_client_handshake(remote, LRMD_CLIENT_HANDSHAKE_TIMEOUT);
1326 }
1327 
1328 /*!
1329  * \internal
1330  * \brief Add trigger and file descriptor mainloop sources for TLS
1331  *
1332  * \param[in] lrmd          API connection with established TLS session
1333  * \param[in] do_handshake  Whether to perform executor handshake
1334  *
1335  * \return Standard Pacemaker return code
1336  */
1337 static int
1338 add_tls_to_mainloop(lrmd_t *lrmd, bool do_handshake)
     /*  */
1339 {
1340     lrmd_private_t *native = lrmd->lrmd_private;
1341     int rc = pcmk_rc_ok;
1342 
1343     char *name = crm_strdup_printf("pacemaker-remote-%s:%d",
1344                                    native->server, native->port);
1345 
1346     struct mainloop_fd_callbacks tls_fd_callbacks = {
1347         .dispatch = lrmd_tls_dispatch,
1348         .destroy = lrmd_tls_connection_destroy,
1349     };
1350 
1351     native->process_notify = mainloop_add_trigger(G_PRIORITY_HIGH,
1352                                                   lrmd_tls_dispatch, lrmd);
1353     native->source = mainloop_add_fd(name, G_PRIORITY_HIGH, native->sock, lrmd,
1354                                      &tls_fd_callbacks);
1355 
1356     /* Async connections lose the client name provided by the API caller, so we
1357      * have to use our generated name here to perform the executor handshake.
1358      *
1359      * @TODO Keep track of the caller-provided name. Perhaps we should be using
1360      * that name in this function instead of generating one anyway.
1361      */
1362     if (do_handshake) {
1363         rc = lrmd_handshake(lrmd, name);
1364         rc = pcmk_legacy2rc(rc);
1365     }
1366     free(name);
1367     return rc;
1368 }
1369 
1370 static void
1371 lrmd_tcp_connect_cb(void *userdata, int rc, int sock)
     /*  */
1372 {
1373     lrmd_t *lrmd = userdata;
1374     lrmd_private_t *native = lrmd->lrmd_private;
1375     gnutls_datum_t psk_key = { NULL, 0 };
1376 
1377     native->async_timer = 0;
1378 
1379     if (rc != pcmk_rc_ok) {
1380         lrmd_tls_connection_destroy(lrmd);
1381         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1382                  CRM_XS " rc=%d",
1383                  native->server, native->port, pcmk_rc_str(rc), rc);
1384         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1385         return;
1386     }
1387 
1388     /* The TCP connection was successful, so establish the TLS connection.
1389      * @TODO make this async to avoid blocking code in client
1390      */
1391 
1392     native->sock = sock;
1393 
1394     rc = lrmd__init_remote_key(&psk_key);
1395     if (rc != pcmk_rc_ok) {
1396         crm_info("Could not connect to Pacemaker Remote at %s:%d: %s "
1397                  CRM_XS " rc=%d",
1398                  native->server, native->port, pcmk_rc_str(rc), rc);
1399         lrmd_tls_connection_destroy(lrmd);
1400         report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1401         return;
1402     }
1403 
1404     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1405     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1406     gnutls_free(psk_key.data);
1407 
1408     native->remote->tls_session = pcmk__new_tls_session(sock, GNUTLS_CLIENT,
1409                                                         GNUTLS_CRD_PSK,
1410                                                         native->psk_cred_c);
1411     if (native->remote->tls_session == NULL) {
1412         lrmd_tls_connection_destroy(lrmd);
1413         report_async_connection_result(lrmd, -EPROTO);
1414         return;
1415     }
1416 
1417     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1418         crm_warn("Disconnecting after TLS handshake with Pacemaker Remote server %s:%d failed",
1419                  native->server, native->port);
1420         gnutls_deinit(*native->remote->tls_session);
1421         gnutls_free(native->remote->tls_session);
1422         native->remote->tls_session = NULL;
1423         lrmd_tls_connection_destroy(lrmd);
1424         report_async_connection_result(lrmd, -EKEYREJECTED);
1425         return;
1426     }
1427 
1428     crm_info("TLS connection to Pacemaker Remote server %s:%d succeeded",
1429              native->server, native->port);
1430     rc = add_tls_to_mainloop(lrmd, true);
1431     report_async_connection_result(lrmd, pcmk_rc2legacy(rc));
1432 }
1433 
1434 static int
1435 lrmd_tls_connect_async(lrmd_t * lrmd, int timeout /*ms */ )
     /*  */
1436 {
1437     int rc;
1438     int timer_id = 0;
1439     lrmd_private_t *native = lrmd->lrmd_private;
1440 
1441     lrmd_gnutls_global_init();
1442     native->sock = -1;
1443     rc = pcmk__connect_remote(native->server, native->port, timeout, &timer_id,
1444                               &(native->sock), lrmd, lrmd_tcp_connect_cb);
1445     if (rc != pcmk_rc_ok) {
1446         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
1447                  CRM_XS " rc=%d",
1448                  native->server, native->port, pcmk_rc_str(rc), rc);
1449         return pcmk_rc2legacy(rc);
1450     }
1451     native->async_timer = timer_id;
1452     return pcmk_ok;
1453 }
1454 
1455 static int
1456 lrmd_tls_connect(lrmd_t * lrmd, int *fd)
     /*  */
1457 {
1458     int rc;
1459 
1460     lrmd_private_t *native = lrmd->lrmd_private;
1461     gnutls_datum_t psk_key = { NULL, 0 };
1462 
1463     lrmd_gnutls_global_init();
1464 
1465     native->sock = -1;
1466     rc = pcmk__connect_remote(native->server, native->port, 0, NULL,
1467                               &(native->sock), NULL, NULL);
1468     if (rc != pcmk_rc_ok) {
1469         crm_warn("Pacemaker Remote connection to %s:%s failed: %s "
1470                  CRM_XS " rc=%d",
1471                  native->server, native->port, pcmk_rc_str(rc), rc);
1472         lrmd_tls_connection_destroy(lrmd);
1473         return -ENOTCONN;
1474     }
1475 
1476     rc = lrmd__init_remote_key(&psk_key);
1477     if (rc != pcmk_rc_ok) {
1478         lrmd_tls_connection_destroy(lrmd);
1479         return pcmk_rc2legacy(rc);
1480     }
1481 
1482     gnutls_psk_allocate_client_credentials(&native->psk_cred_c);
1483     gnutls_psk_set_client_credentials(native->psk_cred_c, DEFAULT_REMOTE_USERNAME, &psk_key, GNUTLS_PSK_KEY_RAW);
1484     gnutls_free(psk_key.data);
1485 
1486     native->remote->tls_session = pcmk__new_tls_session(native->sock, GNUTLS_CLIENT,
1487                                                         GNUTLS_CRD_PSK,
1488                                                         native->psk_cred_c);
1489     if (native->remote->tls_session == NULL) {
1490         lrmd_tls_connection_destroy(lrmd);
1491         return -EPROTO;
1492     }
1493 
1494     if (lrmd__tls_client_handshake(native->remote) != pcmk_rc_ok) {
1495         crm_err("Session creation for %s:%d failed", native->server, native->port);
1496         gnutls_deinit(*native->remote->tls_session);
1497         gnutls_free(native->remote->tls_session);
1498         native->remote->tls_session = NULL;
1499         lrmd_tls_connection_destroy(lrmd);
1500         return -EKEYREJECTED;
1501     }
1502 
1503     crm_info("Client TLS connection established with Pacemaker Remote server %s:%d", native->server,
1504              native->port);
1505 
1506     if (fd) {
1507         *fd = native->sock;
1508     } else {
1509         add_tls_to_mainloop(lrmd, false);
1510     }
1511     return pcmk_ok;
1512 }
1513 #endif
1514 
1515 static int
1516 lrmd_api_connect(lrmd_t * lrmd, const char *name, int *fd)
     /*  */
1517 {
1518     int rc = -ENOTCONN;
1519     lrmd_private_t *native = lrmd->lrmd_private;
1520 
1521     switch (native->type) {
1522         case pcmk__client_ipc:
1523             rc = lrmd_ipc_connect(lrmd, fd);
1524             break;
1525 #ifdef HAVE_GNUTLS_GNUTLS_H
1526         case pcmk__client_tls:
1527             rc = lrmd_tls_connect(lrmd, fd);
1528             break;
1529 #endif
1530         default:
1531             crm_err("Unsupported executor connection type (bug?): %d",
1532                     native->type);
1533             rc = -EPROTONOSUPPORT;
1534     }
1535 
1536     if (rc == pcmk_ok) {
1537         rc = lrmd_handshake(lrmd, name);
1538     }
1539 
1540     return rc;
1541 }
1542 
1543 static int
1544 lrmd_api_connect_async(lrmd_t * lrmd, const char *name, int timeout)
     /*  */
1545 {
1546     int rc = pcmk_ok;
1547     lrmd_private_t *native = lrmd->lrmd_private;
1548 
1549     CRM_CHECK(native && native->callback, return -EINVAL);
1550 
1551     switch (native->type) {
1552         case pcmk__client_ipc:
1553             /* fake async connection with ipc.  it should be fast
1554              * enough that we gain very little from async */
1555             rc = lrmd_api_connect(lrmd, name, NULL);
1556             if (!rc) {
1557                 report_async_connection_result(lrmd, rc);
1558             }
1559             break;
1560 #ifdef HAVE_GNUTLS_GNUTLS_H
1561         case pcmk__client_tls:
1562             rc = lrmd_tls_connect_async(lrmd, timeout);
1563             if (rc) {
1564                 /* connection failed, report rc now */
1565                 report_async_connection_result(lrmd, rc);
1566             }
1567             break;
1568 #endif
1569         default:
1570             crm_err("Unsupported executor connection type (bug?): %d",
1571                     native->type);
1572             rc = -EPROTONOSUPPORT;
1573     }
1574 
1575     return rc;
1576 }
1577 
1578 static void
1579 lrmd_ipc_disconnect(lrmd_t * lrmd)
     /*  */
1580 {
1581     lrmd_private_t *native = lrmd->lrmd_private;
1582 
1583     if (native->source != NULL) {
1584         /* Attached to mainloop */
1585         mainloop_del_ipc_client(native->source);
1586         native->source = NULL;
1587         native->ipc = NULL;
1588 
1589     } else if (native->ipc) {
1590         /* Not attached to mainloop */
1591         crm_ipc_t *ipc = native->ipc;
1592 
1593         native->ipc = NULL;
1594         crm_ipc_close(ipc);
1595         crm_ipc_destroy(ipc);
1596     }
1597 }
1598 
1599 #ifdef HAVE_GNUTLS_GNUTLS_H
1600 static void
1601 lrmd_tls_disconnect(lrmd_t * lrmd)
     /*  */
1602 {
1603     lrmd_private_t *native = lrmd->lrmd_private;
1604 
1605     if (native->remote->tls_session) {
1606         gnutls_bye(*native->remote->tls_session, GNUTLS_SHUT_RDWR);
1607         gnutls_deinit(*native->remote->tls_session);
1608         gnutls_free(native->remote->tls_session);
1609         native->remote->tls_session = 0;
1610     }
1611 
1612     if (native->async_timer) {
1613         g_source_remove(native->async_timer);
1614         native->async_timer = 0;
1615     }
1616 
1617     if (native->source != NULL) {
1618         /* Attached to mainloop */
1619         mainloop_del_ipc_client(native->source);
1620         native->source = NULL;
1621 
1622     } else if (native->sock) {
1623         close(native->sock);
1624         native->sock = 0;
1625     }
1626 
1627     if (native->pending_notify) {
1628         g_list_free_full(native->pending_notify, lrmd_free_xml);
1629         native->pending_notify = NULL;
1630     }
1631 }
1632 #endif
1633 
1634 static int
1635 lrmd_api_disconnect(lrmd_t * lrmd)
     /*  */
1636 {
1637     lrmd_private_t *native = lrmd->lrmd_private;
1638     int rc = pcmk_ok;
1639 
1640     crm_info("Disconnecting %s %s executor connection",
1641              pcmk__client_type_str(native->type),
1642              (native->remote_nodename? native->remote_nodename : "local"));
1643     switch (native->type) {
1644         case pcmk__client_ipc:
1645             lrmd_ipc_disconnect(lrmd);
1646             break;
1647 #ifdef HAVE_GNUTLS_GNUTLS_H
1648         case pcmk__client_tls:
1649             lrmd_tls_disconnect(lrmd);
1650             break;
1651 #endif
1652         default:
1653             crm_err("Unsupported executor connection type (bug?): %d",
1654                     native->type);
1655             rc = -EPROTONOSUPPORT;
1656     }
1657 
1658     free(native->token);
1659     native->token = NULL;
1660 
1661     free(native->peer_version);
1662     native->peer_version = NULL;
1663     return rc;
1664 }
1665 
1666 static int
1667 lrmd_api_register_rsc(lrmd_t * lrmd,
     /*  */
1668                       const char *rsc_id,
1669                       const char *class,
1670                       const char *provider, const char *type, enum lrmd_call_options options)
1671 {
1672     int rc = pcmk_ok;
1673     xmlNode *data = NULL;
1674 
1675     if (!class || !type || !rsc_id) {
1676         return -EINVAL;
1677     }
1678     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1679         && (provider == NULL)) {
1680         return -EINVAL;
1681     }
1682 
1683     data = create_xml_node(NULL, F_LRMD_RSC);
1684 
1685     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1686     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1687     crm_xml_add(data, F_LRMD_CLASS, class);
1688     crm_xml_add(data, F_LRMD_PROVIDER, provider);
1689     crm_xml_add(data, F_LRMD_TYPE, type);
1690     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_REG, data, NULL, 0, options, TRUE);
1691     free_xml(data);
1692 
1693     return rc;
1694 }
1695 
1696 static int
1697 lrmd_api_unregister_rsc(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1698 {
1699     int rc = pcmk_ok;
1700     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1701 
1702     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1703     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1704     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_UNREG, data, NULL, 0, options, TRUE);
1705     free_xml(data);
1706 
1707     return rc;
1708 }
1709 
1710 lrmd_rsc_info_t *
1711 lrmd_new_rsc_info(const char *rsc_id, const char *standard,
     /*  */
1712                   const char *provider, const char *type)
1713 {
1714     lrmd_rsc_info_t *rsc_info = calloc(1, sizeof(lrmd_rsc_info_t));
1715 
1716     CRM_ASSERT(rsc_info);
1717     if (rsc_id) {
1718         rsc_info->id = strdup(rsc_id);
1719         CRM_ASSERT(rsc_info->id);
1720     }
1721     if (standard) {
1722         rsc_info->standard = strdup(standard);
1723         CRM_ASSERT(rsc_info->standard);
1724     }
1725     if (provider) {
1726         rsc_info->provider = strdup(provider);
1727         CRM_ASSERT(rsc_info->provider);
1728     }
1729     if (type) {
1730         rsc_info->type = strdup(type);
1731         CRM_ASSERT(rsc_info->type);
1732     }
1733     return rsc_info;
1734 }
1735 
1736 lrmd_rsc_info_t *
1737 lrmd_copy_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1738 {
1739     return lrmd_new_rsc_info(rsc_info->id, rsc_info->standard,
1740                              rsc_info->provider, rsc_info->type);
1741 }
1742 
1743 void
1744 lrmd_free_rsc_info(lrmd_rsc_info_t * rsc_info)
     /*  */
1745 {
1746     if (!rsc_info) {
1747         return;
1748     }
1749     free(rsc_info->id);
1750     free(rsc_info->type);
1751     free(rsc_info->standard);
1752     free(rsc_info->provider);
1753     free(rsc_info);
1754 }
1755 
1756 static lrmd_rsc_info_t *
1757 lrmd_api_get_rsc_info(lrmd_t * lrmd, const char *rsc_id, enum lrmd_call_options options)
     /*  */
1758 {
1759     lrmd_rsc_info_t *rsc_info = NULL;
1760     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1761     xmlNode *output = NULL;
1762     const char *class = NULL;
1763     const char *provider = NULL;
1764     const char *type = NULL;
1765 
1766     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1767     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1768     lrmd_send_command(lrmd, LRMD_OP_RSC_INFO, data, &output, 0, options, TRUE);
1769     free_xml(data);
1770 
1771     if (!output) {
1772         return NULL;
1773     }
1774 
1775     class = crm_element_value(output, F_LRMD_CLASS);
1776     provider = crm_element_value(output, F_LRMD_PROVIDER);
1777     type = crm_element_value(output, F_LRMD_TYPE);
1778 
1779     if (!class || !type) {
1780         free_xml(output);
1781         return NULL;
1782     } else if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_provider)
1783                && !provider) {
1784         free_xml(output);
1785         return NULL;
1786     }
1787 
1788     rsc_info = lrmd_new_rsc_info(rsc_id, class, provider, type);
1789     free_xml(output);
1790     return rsc_info;
1791 }
1792 
1793 void
1794 lrmd_free_op_info(lrmd_op_info_t *op_info)
     /*  */
1795 {
1796     if (op_info) {
1797         free(op_info->rsc_id);
1798         free(op_info->action);
1799         free(op_info->interval_ms_s);
1800         free(op_info->timeout_ms_s);
1801         free(op_info);
1802     }
1803 }
1804 
1805 static int
1806 lrmd_api_get_recurring_ops(lrmd_t *lrmd, const char *rsc_id, int timeout_ms,
     /*  */
1807                            enum lrmd_call_options options, GList **output)
1808 {
1809     xmlNode *data = NULL;
1810     xmlNode *output_xml = NULL;
1811     int rc = pcmk_ok;
1812 
1813     if (output == NULL) {
1814         return -EINVAL;
1815     }
1816     *output = NULL;
1817 
1818     // Send request
1819     if (rsc_id) {
1820         data = create_xml_node(NULL, F_LRMD_RSC);
1821         crm_xml_add(data, F_LRMD_ORIGIN, __func__);
1822         crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
1823     }
1824     rc = lrmd_send_command(lrmd, LRMD_OP_GET_RECURRING, data, &output_xml,
1825                            timeout_ms, options, TRUE);
1826     if (data) {
1827         free_xml(data);
1828     }
1829 
1830     // Process reply
1831     if ((rc != pcmk_ok) || (output_xml == NULL)) {
1832         return rc;
1833     }
1834     for (xmlNode *rsc_xml = first_named_child(output_xml, F_LRMD_RSC);
1835          (rsc_xml != NULL) && (rc == pcmk_ok);
1836          rsc_xml = crm_next_same_xml(rsc_xml)) {
1837 
1838         rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1839         if (rsc_id == NULL) {
1840             crm_err("Could not parse recurring operation information from executor");
1841             continue;
1842         }
1843         for (xmlNode *op_xml = first_named_child(rsc_xml, T_LRMD_RSC_OP);
1844              op_xml != NULL; op_xml = crm_next_same_xml(op_xml)) {
1845 
1846             lrmd_op_info_t *op_info = calloc(1, sizeof(lrmd_op_info_t));
1847 
1848             if (op_info == NULL) {
1849                 rc = -ENOMEM;
1850                 break;
1851             }
1852             op_info->rsc_id = strdup(rsc_id);
1853             op_info->action = crm_element_value_copy(op_xml, F_LRMD_RSC_ACTION);
1854             op_info->interval_ms_s = crm_element_value_copy(op_xml,
1855                                                             F_LRMD_RSC_INTERVAL);
1856             op_info->timeout_ms_s = crm_element_value_copy(op_xml,
1857                                                            F_LRMD_TIMEOUT);
1858             *output = g_list_prepend(*output, op_info);
1859         }
1860     }
1861     free_xml(output_xml);
1862     return rc;
1863 }
1864 
1865 
1866 static void
1867 lrmd_api_set_callback(lrmd_t * lrmd, lrmd_event_callback callback)
     /*  */
1868 {
1869     lrmd_private_t *native = lrmd->lrmd_private;
1870 
1871     native->callback = callback;
1872 }
1873 
1874 void
1875 lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg))
     /*  */
1876 {
1877     lrmd_private_t *native = lrmd->lrmd_private;
1878 
1879     native->proxy_callback = callback;
1880     native->proxy_callback_userdata = userdata;
1881 }
1882 
1883 void
1884 lrmd_internal_proxy_dispatch(lrmd_t *lrmd, xmlNode *msg)
     /*  */
1885 {
1886     lrmd_private_t *native = lrmd->lrmd_private;
1887 
1888     if (native->proxy_callback) {
1889         crm_log_xml_trace(msg, "PROXY_INBOUND");
1890         native->proxy_callback(lrmd, native->proxy_callback_userdata, msg);
1891     }
1892 }
1893 
1894 int
1895 lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg)
     /*  */
1896 {
1897     if (lrmd == NULL) {
1898         return -ENOTCONN;
1899     }
1900     crm_xml_add(msg, F_LRMD_OPERATION, CRM_OP_IPC_FWD);
1901 
1902     crm_log_xml_trace(msg, "PROXY_OUTBOUND");
1903     return lrmd_send_xml_no_reply(lrmd, msg);
1904 }
1905 
1906 static int
1907 stonith_get_metadata(const char *provider, const char *type, char **output)
     /*  */
1908 {
1909     int rc = pcmk_ok;
1910     stonith_t *stonith_api = stonith_api_new();
1911 
1912     if (stonith_api == NULL) {
1913         crm_err("Could not get fence agent meta-data: API memory allocation failed");
1914         return -ENOMEM;
1915     }
1916 
1917     rc = stonith_api->cmds->metadata(stonith_api, st_opt_sync_call, type,
1918                                      provider, output, 0);
1919     if ((rc == pcmk_ok) && (*output == NULL)) {
1920         rc = -EIO;
1921     }
1922     stonith_api->cmds->free(stonith_api);
1923     return rc;
1924 }
1925 
1926 static int
1927 lrmd_api_get_metadata(lrmd_t *lrmd, const char *standard, const char *provider,
     /*  */
1928                       const char *type, char **output,
1929                       enum lrmd_call_options options)
1930 {
1931     return lrmd->cmds->get_metadata_params(lrmd, standard, provider, type,
1932                                            output, options, NULL);
1933 }
1934 
1935 static int
1936 lrmd_api_get_metadata_params(lrmd_t *lrmd, const char *standard,
     /*  */
1937                              const char *provider, const char *type,
1938                              char **output, enum lrmd_call_options options,
1939                              lrmd_key_value_t *params)
1940 {
1941     svc_action_t *action = NULL;
1942     GHashTable *params_table = NULL;
1943 
1944     if (!standard || !type) {
1945         lrmd_key_value_freeall(params);
1946         return -EINVAL;
1947     }
1948 
1949     if (pcmk__str_eq(standard, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
1950         lrmd_key_value_freeall(params);
1951         return stonith_get_metadata(provider, type, output);
1952     }
1953 
1954     params_table = pcmk__strkey_table(free, free);
1955     for (const lrmd_key_value_t *param = params; param; param = param->next) {
1956         g_hash_table_insert(params_table, strdup(param->key), strdup(param->value));
1957     }
1958     action = resources_action_create(type, standard, provider, type,
1959                                      CRMD_ACTION_METADATA, 0,
1960                                      CRMD_METADATA_CALL_TIMEOUT, params_table,
1961                                      0);
1962     lrmd_key_value_freeall(params);
1963 
1964     if (action == NULL) {
1965         crm_err("Unable to retrieve meta-data for %s:%s:%s",
1966                 standard, provider, type);
1967         return -EINVAL;
1968     }
1969 
1970     if (!services_action_sync(action)) {
1971         crm_err("Failed to retrieve meta-data for %s:%s:%s",
1972                 standard, provider, type);
1973         services_action_free(action);
1974         return -EIO;
1975     }
1976 
1977     if (!action->stdout_data) {
1978         crm_err("Failed to receive meta-data for %s:%s:%s",
1979                 standard, provider, type);
1980         services_action_free(action);
1981         return -EIO;
1982     }
1983 
1984     *output = strdup(action->stdout_data);
1985     services_action_free(action);
1986 
1987     return pcmk_ok;
1988 }
1989 
1990 static int
1991 lrmd_api_exec(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
1992               const char *userdata, guint interval_ms,
1993               int timeout,      /* ms */
1994               int start_delay,  /* ms */
1995               enum lrmd_call_options options, lrmd_key_value_t * params)
1996 {
1997     int rc = pcmk_ok;
1998     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
1999     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2000     lrmd_key_value_t *tmp = NULL;
2001 
2002     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2003     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2004     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2005     crm_xml_add(data, F_LRMD_RSC_USERDATA_STR, userdata);
2006     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2007     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2008     crm_xml_add_int(data, F_LRMD_RSC_START_DELAY, start_delay);
2009 
2010     for (tmp = params; tmp; tmp = tmp->next) {
2011         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2012     }
2013 
2014     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_EXEC, data, NULL, timeout, options, TRUE);
2015     free_xml(data);
2016 
2017     lrmd_key_value_freeall(params);
2018     return rc;
2019 }
2020 
2021 /* timeout is in ms */
2022 static int
2023 lrmd_api_exec_alert(lrmd_t *lrmd, const char *alert_id, const char *alert_path,
     /*  */
2024                     int timeout, lrmd_key_value_t *params)
2025 {
2026     int rc = pcmk_ok;
2027     xmlNode *data = create_xml_node(NULL, F_LRMD_ALERT);
2028     xmlNode *args = create_xml_node(data, XML_TAG_ATTRS);
2029     lrmd_key_value_t *tmp = NULL;
2030 
2031     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2032     crm_xml_add(data, F_LRMD_ALERT_ID, alert_id);
2033     crm_xml_add(data, F_LRMD_ALERT_PATH, alert_path);
2034     crm_xml_add_int(data, F_LRMD_TIMEOUT, timeout);
2035 
2036     for (tmp = params; tmp; tmp = tmp->next) {
2037         hash2smartfield((gpointer) tmp->key, (gpointer) tmp->value, args);
2038     }
2039 
2040     rc = lrmd_send_command(lrmd, LRMD_OP_ALERT_EXEC, data, NULL, timeout,
2041                            lrmd_opt_notify_orig_only, TRUE);
2042     free_xml(data);
2043 
2044     lrmd_key_value_freeall(params);
2045     return rc;
2046 }
2047 
2048 static int
2049 lrmd_api_cancel(lrmd_t *lrmd, const char *rsc_id, const char *action,
     /*  */
2050                 guint interval_ms)
2051 {
2052     int rc = pcmk_ok;
2053     xmlNode *data = create_xml_node(NULL, F_LRMD_RSC);
2054 
2055     crm_xml_add(data, F_LRMD_ORIGIN, __func__);
2056     crm_xml_add(data, F_LRMD_RSC_ACTION, action);
2057     crm_xml_add(data, F_LRMD_RSC_ID, rsc_id);
2058     crm_xml_add_ms(data, F_LRMD_RSC_INTERVAL, interval_ms);
2059     rc = lrmd_send_command(lrmd, LRMD_OP_RSC_CANCEL, data, NULL, 0, 0, TRUE);
2060     free_xml(data);
2061     return rc;
2062 }
2063 
2064 static int
2065 list_stonith_agents(lrmd_list_t ** resources)
     /*  */
2066 {
2067     int rc = 0;
2068     stonith_t *stonith_api = stonith_api_new();
2069     stonith_key_value_t *stonith_resources = NULL;
2070     stonith_key_value_t *dIter = NULL;
2071 
2072     if (stonith_api == NULL) {
2073         crm_err("Could not list fence agents: API memory allocation failed");
2074         return -ENOMEM;
2075     }
2076     stonith_api->cmds->list_agents(stonith_api, st_opt_sync_call, NULL,
2077                                    &stonith_resources, 0);
2078     stonith_api->cmds->free(stonith_api);
2079 
2080     for (dIter = stonith_resources; dIter; dIter = dIter->next) {
2081         rc++;
2082         if (resources) {
2083             *resources = lrmd_list_add(*resources, dIter->value);
2084         }
2085     }
2086 
2087     stonith_key_value_freeall(stonith_resources, 1, 0);
2088     return rc;
2089 }
2090 
2091 static int
2092 lrmd_api_list_agents(lrmd_t * lrmd, lrmd_list_t ** resources, const char *class,
     /*  */
2093                      const char *provider)
2094 {
2095     int rc = 0;
2096     int stonith_count = 0; // Initially, whether to include stonith devices
2097 
2098     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_casei)) {
2099         stonith_count = 1;
2100 
2101     } else {
2102         GList *gIter = NULL;
2103         GList *agents = resources_list_agents(class, provider);
2104 
2105         for (gIter = agents; gIter != NULL; gIter = gIter->next) {
2106             *resources = lrmd_list_add(*resources, (const char *)gIter->data);
2107             rc++;
2108         }
2109         g_list_free_full(agents, free);
2110 
2111         if (!class) {
2112             stonith_count = 1;
2113         }
2114     }
2115 
2116     if (stonith_count) {
2117         // Now, if stonith devices are included, how many there are
2118         stonith_count = list_stonith_agents(resources);
2119         if (stonith_count > 0) {
2120             rc += stonith_count;
2121         }
2122     }
2123     if (rc == 0) {
2124         crm_notice("No agents found for class %s", class);
2125         rc = -EPROTONOSUPPORT;
2126     }
2127     return rc;
2128 }
2129 
2130 static bool
2131 does_provider_have_agent(const char *agent, const char *provider, const char *class)
     /*  */
2132 {
2133     bool found = false;
2134     GList *agents = NULL;
2135     GList *gIter2 = NULL;
2136 
2137     agents = resources_list_agents(class, provider);
2138     for (gIter2 = agents; gIter2 != NULL; gIter2 = gIter2->next) {
2139         if (pcmk__str_eq(agent, gIter2->data, pcmk__str_casei)) {
2140             found = true;
2141         }
2142     }
2143     g_list_free_full(agents, free);
2144     return found;
2145 }
2146 
2147 static int
2148 lrmd_api_list_ocf_providers(lrmd_t * lrmd, const char *agent, lrmd_list_t ** providers)
     /*  */
2149 {
2150     int rc = pcmk_ok;
2151     char *provider = NULL;
2152     GList *ocf_providers = NULL;
2153     GList *gIter = NULL;
2154 
2155     ocf_providers = resources_list_providers(PCMK_RESOURCE_CLASS_OCF);
2156 
2157     for (gIter = ocf_providers; gIter != NULL; gIter = gIter->next) {
2158         provider = gIter->data;
2159         if (!agent || does_provider_have_agent(agent, provider,
2160                                                PCMK_RESOURCE_CLASS_OCF)) {
2161             *providers = lrmd_list_add(*providers, (const char *)gIter->data);
2162             rc++;
2163         }
2164     }
2165 
2166     g_list_free_full(ocf_providers, free);
2167     return rc;
2168 }
2169 
2170 static int
2171 lrmd_api_list_standards(lrmd_t * lrmd, lrmd_list_t ** supported)
     /*  */
2172 {
2173     int rc = 0;
2174     GList *standards = NULL;
2175     GList *gIter = NULL;
2176 
2177     standards = resources_list_standards();
2178 
2179     for (gIter = standards; gIter != NULL; gIter = gIter->next) {
2180         *supported = lrmd_list_add(*supported, (const char *)gIter->data);
2181         rc++;
2182     }
2183 
2184     if (list_stonith_agents(NULL) > 0) {
2185         *supported = lrmd_list_add(*supported, PCMK_RESOURCE_CLASS_STONITH);
2186         rc++;
2187     }
2188 
2189     g_list_free_full(standards, free);
2190     return rc;
2191 }
2192 
2193 lrmd_t *
2194 lrmd_api_new(void)
     /*  */
2195 {
2196     lrmd_t *new_lrmd = NULL;
2197     lrmd_private_t *pvt = NULL;
2198 
2199     new_lrmd = calloc(1, sizeof(lrmd_t));
2200     pvt = calloc(1, sizeof(lrmd_private_t));
2201     pvt->remote = calloc(1, sizeof(pcmk__remote_t));
2202     new_lrmd->cmds = calloc(1, sizeof(lrmd_api_operations_t));
2203 
2204     pvt->type = pcmk__client_ipc;
2205     new_lrmd->lrmd_private = pvt;
2206 
2207     new_lrmd->cmds->connect = lrmd_api_connect;
2208     new_lrmd->cmds->connect_async = lrmd_api_connect_async;
2209     new_lrmd->cmds->is_connected = lrmd_api_is_connected;
2210     new_lrmd->cmds->poke_connection = lrmd_api_poke_connection;
2211     new_lrmd->cmds->disconnect = lrmd_api_disconnect;
2212     new_lrmd->cmds->register_rsc = lrmd_api_register_rsc;
2213     new_lrmd->cmds->unregister_rsc = lrmd_api_unregister_rsc;
2214     new_lrmd->cmds->get_rsc_info = lrmd_api_get_rsc_info;
2215     new_lrmd->cmds->get_recurring_ops = lrmd_api_get_recurring_ops;
2216     new_lrmd->cmds->set_callback = lrmd_api_set_callback;
2217     new_lrmd->cmds->get_metadata = lrmd_api_get_metadata;
2218     new_lrmd->cmds->exec = lrmd_api_exec;
2219     new_lrmd->cmds->cancel = lrmd_api_cancel;
2220     new_lrmd->cmds->list_agents = lrmd_api_list_agents;
2221     new_lrmd->cmds->list_ocf_providers = lrmd_api_list_ocf_providers;
2222     new_lrmd->cmds->list_standards = lrmd_api_list_standards;
2223     new_lrmd->cmds->exec_alert = lrmd_api_exec_alert;
2224     new_lrmd->cmds->get_metadata_params = lrmd_api_get_metadata_params;
2225 
2226     return new_lrmd;
2227 }
2228 
2229 lrmd_t *
2230 lrmd_remote_api_new(const char *nodename, const char *server, int port)
     /*  */
2231 {
2232 #ifdef HAVE_GNUTLS_GNUTLS_H
2233     lrmd_t *new_lrmd = lrmd_api_new();
2234     lrmd_private_t *native = new_lrmd->lrmd_private;
2235 
2236     if (!nodename && !server) {
2237         lrmd_api_delete(new_lrmd);
2238         return NULL;
2239     }
2240 
2241     native->type = pcmk__client_tls;
2242     native->remote_nodename = nodename ? strdup(nodename) : strdup(server);
2243     native->server = server ? strdup(server) : strdup(nodename);
2244     native->port = port;
2245     if (native->port == 0) {
2246         native->port = crm_default_remote_port();
2247     }
2248 
2249     return new_lrmd;
2250 #else
2251     crm_err("Cannot communicate with Pacemaker Remote because GnuTLS is not enabled for this build");
2252     return NULL;
2253 #endif
2254 }
2255 
2256 void
2257 lrmd_api_delete(lrmd_t * lrmd)
     /*  */
2258 {
2259     if (!lrmd) {
2260         return;
2261     }
2262     lrmd->cmds->disconnect(lrmd);       /* no-op if already disconnected */
2263     free(lrmd->cmds);
2264     if (lrmd->lrmd_private) {
2265         lrmd_private_t *native = lrmd->lrmd_private;
2266 
2267 #ifdef HAVE_GNUTLS_GNUTLS_H
2268         free(native->server);
2269 #endif
2270         free(native->remote_nodename);
2271         free(native->remote);
2272         free(native->token);
2273         free(native->peer_version);
2274     }
2275 
2276     free(lrmd->lrmd_private);
2277     free(lrmd);
2278 }
/* */
root/lib/lrmd/lrmd_client.c

DEFINITIONS
