daemons/based/based

/* */
This source file includes following definitions.
cib_legacy_mode
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
create_cib_reply
do_local_notify
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
local_notify_destroy_callback
check_local_notify
queue_local_notify
parse_local_options_v1
parse_local_options_v2
parse_local_options
parse_peer_options_v1
parse_peer_options_v2
parse_peer_options
forward_request
send_peer_reply
cib_process_request
prepare_input
contains_config_change
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
initiate_exit
cib_shutdown
terminate_cib
   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>   // PRIu64
  22 
  23 #include <glib.h>
  24 #include <libxml/tree.h>
  25 
  26 #include <crm/crm.h>
  27 #include <crm/cib.h>
  28 #include <crm/msg_xml.h>
  29 #include <crm/cluster/internal.h>
  30 
  31 #include <crm/common/xml.h>
  32 #include <crm/common/remote_internal.h>
  33 
  34 #include <pacemaker-based.h>
  35 
  36 #define EXIT_ESCALATION_MS 10000
  37 
  38 static unsigned long cib_local_bcast_num = 0;
  39 
  40 typedef struct cib_local_notify_s {
  41     xmlNode *notify_src;
  42     char *client_id;
  43     gboolean from_peer;
  44     gboolean sync_reply;
  45 } cib_local_notify_t;
  46 
  47 int next_client_id = 0;
  48 
  49 gboolean legacy_mode = FALSE;
  50 
  51 qb_ipcs_service_t *ipcs_ro = NULL;
  52 qb_ipcs_service_t *ipcs_rw = NULL;
  53 qb_ipcs_service_t *ipcs_shm = NULL;
  54 
  55 static int cib_process_command(xmlNode *request,
  56                                const cib__operation_t *operation,
  57                                cib__op_fn_t op_function, xmlNode **reply,
  58                                xmlNode **cib_diff, bool privileged);
  59 
  60 static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
  61                                     size_t size, gboolean privileged);
  62 
  63 gboolean
  64 cib_legacy_mode(void)
     /*  */
  65 {
  66     return legacy_mode;
  67 }
  68 
  69 static int32_t
  70 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  71 {
  72     if (cib_shutdown_flag) {
  73         crm_info("Ignoring new IPC client [%d] during shutdown",
  74                  pcmk__client_pid(c));
  75         return -EPERM;
  76     }
  77 
  78     if (pcmk__new_client(c, uid, gid) == NULL) {
  79         return -EIO;
  80     }
  81     return 0;
  82 }
  83 
  84 static int32_t
  85 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  86 {
  87     pcmk__client_t *client = pcmk__find_client(c);
  88 
  89     crm_trace("%p message from %s", c, client->id);
  90     return cib_common_callback(c, data, size, TRUE);
  91 }
  92 
  93 static int32_t
  94 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  95 {
  96     pcmk__client_t *client = pcmk__find_client(c);
  97 
  98     crm_trace("%p message from %s", c, client->id);
  99     return cib_common_callback(c, data, size, FALSE);
 100 }
 101 
 102 /* Error code means? */
 103 static int32_t
 104 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 105 {
 106     pcmk__client_t *client = pcmk__find_client(c);
 107 
 108     if (client == NULL) {
 109         return 0;
 110     }
 111     crm_trace("Connection %p", c);
 112     pcmk__free_client(client);
 113     return 0;
 114 }
 115 
 116 static void
 117 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 118 {
 119     crm_trace("Connection %p", c);
 120     cib_ipc_closed(c);
 121     if (cib_shutdown_flag) {
 122         cib_shutdown(0);
 123     }
 124 }
 125 
 126 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 127     .connection_accept = cib_ipc_accept,
 128     .connection_created = NULL,
 129     .msg_process = cib_ipc_dispatch_ro,
 130     .connection_closed = cib_ipc_closed,
 131     .connection_destroyed = cib_ipc_destroy
 132 };
 133 
 134 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 135     .connection_accept = cib_ipc_accept,
 136     .connection_created = NULL,
 137     .msg_process = cib_ipc_dispatch_rw,
 138     .connection_closed = cib_ipc_closed,
 139     .connection_destroyed = cib_ipc_destroy
 140 };
 141 
 142 /*!
 143  * \internal
 144  * \brief Create reply XML for a CIB request
 145  *
 146  * \param[in] op            CIB operation type
 147  * \param[in] call_id       CIB call ID
 148  * \param[in] client_id     CIB client ID
 149  * \param[in] call_options  Group of <tt>enum cib_call_options</tt> flags
 150  * \param[in] rc            Request return code
 151  * \param[in] call_data     Request output data
 152  *
 153  * \return Reply XML
 154  *
 155  * \note The caller is responsible for freeing the return value using
 156  *       \p free_xml().
 157  */
 158 static xmlNode *
 159 create_cib_reply(const char *op, const char *call_id, const char *client_id,
     /*  */
 160                  int call_options, int rc, xmlNode *call_data)
 161 {
 162     xmlNode *reply = create_xml_node(NULL, "cib-reply");
 163 
 164     CRM_ASSERT(reply != NULL);
 165 
 166     crm_xml_add(reply, F_TYPE, T_CIB);
 167     crm_xml_add(reply, F_CIB_OPERATION, op);
 168     crm_xml_add(reply, F_CIB_CALLID, call_id);
 169     crm_xml_add(reply, F_CIB_CLIENTID, client_id);
 170     crm_xml_add_int(reply, F_CIB_CALLOPTS, call_options);
 171     crm_xml_add_int(reply, F_CIB_RC, rc);
 172 
 173     if (call_data != NULL) {
 174         crm_trace("Attaching reply output");
 175         add_message_xml(reply, F_CIB_CALLDATA, call_data);
 176     }
 177 
 178     crm_log_xml_explicit(reply, "cib:reply");
 179     return reply;
 180 }
 181 
 182 static void
 183 do_local_notify(const xmlNode *notify_src, const char *client_id,
     /*  */
 184                 bool sync_reply, bool from_peer)
 185 {
 186     int rid = 0;
 187     int call_id = 0;
 188     pcmk__client_t *client_obj = NULL;
 189 
 190     CRM_ASSERT(notify_src && client_id);
 191 
 192     crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
 193 
 194     client_obj = pcmk__find_client_by_id(client_id);
 195     if (client_obj == NULL) {
 196         crm_debug("Could not send response %d: client %s not found",
 197                   call_id, client_id);
 198         return;
 199     }
 200 
 201     if (sync_reply) {
 202         if (client_obj->ipcs) {
 203             CRM_LOG_ASSERT(client_obj->request_id);
 204 
 205             rid = client_obj->request_id;
 206             client_obj->request_id = 0;
 207 
 208             crm_trace("Sending response %d to client %s%s",
 209                       rid, pcmk__client_name(client_obj),
 210                       (from_peer? " (originator of delegated request)" : ""));
 211         } else {
 212             crm_trace("Sending response (call %d) to client %s%s",
 213                       call_id, pcmk__client_name(client_obj),
 214                       (from_peer? " (originator of delegated request)" : ""));
 215         }
 216 
 217     } else {
 218         crm_trace("Sending event %d to client %s%s",
 219                   call_id, pcmk__client_name(client_obj),
 220                   (from_peer? " (originator of delegated request)" : ""));
 221     }
 222 
 223     switch (PCMK__CLIENT_TYPE(client_obj)) {
 224         case pcmk__client_ipc:
 225             {
 226                 int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
 227                                             (sync_reply? crm_ipc_flags_none
 228                                              : crm_ipc_server_event));
 229 
 230                 if (rc != pcmk_rc_ok) {
 231                     crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
 232                              (sync_reply? "Synchronous" : "Asynchronous"),
 233                              pcmk__client_name(client_obj), pcmk_rc_str(rc),
 234                              rc);
 235                 }
 236             }
 237             break;
 238 #ifdef HAVE_GNUTLS_GNUTLS_H
 239         case pcmk__client_tls:
 240 #endif
 241         case pcmk__client_tcp:
 242             pcmk__remote_send_xml(client_obj->remote, notify_src);
 243             break;
 244         default:
 245             crm_err("Unknown transport for client %s "
 246                     CRM_XS " flags=%#016" PRIx64,
 247                     pcmk__client_name(client_obj), client_obj->flags);
 248     }
 249 }
 250 
 251 void
 252 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 253                            pcmk__client_t *cib_client, gboolean privileged)
 254 {
 255     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 256     int call_options = cib_none;
 257 
 258     crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 259 
 260     /* Requests with cib_transaction set should not be sent to based directly
 261      * (outside of a commit-transaction request)
 262      */
 263     if (pcmk_is_set(call_options, cib_transaction)) {
 264         return;
 265     }
 266 
 267     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 268         if (flags & crm_ipc_client_response) {
 269             xmlNode *ack = create_xml_node(NULL, __func__);
 270 
 271             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
 272             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
 273             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 274             cib_client->request_id = 0;
 275             free_xml(ack);
 276         }
 277         return;
 278 
 279     } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
 280         /* Update the notify filters for this client */
 281         int on_off = 0;
 282         crm_exit_t status = CRM_EX_OK;
 283         uint64_t bit = UINT64_C(0);
 284         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 285 
 286         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 287 
 288         crm_debug("Setting %s callbacks %s for client %s",
 289                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 290 
 291         if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
 292             bit = cib_notify_post;
 293 
 294         } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
 295             bit = cib_notify_pre;
 296 
 297         } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
 298             bit = cib_notify_confirm;
 299 
 300         } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
 301             bit = cib_notify_diff;
 302 
 303         } else {
 304             status = CRM_EX_INVALID_PARAM;
 305         }
 306 
 307         if (bit != 0) {
 308             if (on_off) {
 309                 pcmk__set_client_flags(cib_client, bit);
 310             } else {
 311                 pcmk__clear_client_flags(cib_client, bit);
 312             }
 313         }
 314 
 315         pcmk__ipc_send_ack(cib_client, id, flags, "ack", NULL, status);
 316         return;
 317     }
 318 
 319     cib_process_request(op_request, privileged, cib_client);
 320 }
 321 
 322 int32_t
 323 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 324 {
 325     uint32_t id = 0;
 326     uint32_t flags = 0;
 327     int call_options = 0;
 328     pcmk__client_t *cib_client = pcmk__find_client(c);
 329     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 330 
 331     if (op_request) {
 332         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 333     }
 334 
 335     if (op_request == NULL) {
 336         crm_trace("Invalid message from %p", c);
 337         pcmk__ipc_send_ack(cib_client, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
 338         return 0;
 339 
 340     } else if(cib_client == NULL) {
 341         crm_trace("Invalid client %p", c);
 342         return 0;
 343     }
 344 
 345     if (pcmk_is_set(call_options, cib_sync_call)) {
 346         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 347         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 348         cib_client->request_id = id;    /* Reply only to the last one */
 349     }
 350 
 351     if (cib_client->name == NULL) {
 352         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 353 
 354         if (value == NULL) {
 355             cib_client->name = pcmk__itoa(cib_client->pid);
 356         } else {
 357             cib_client->name = strdup(value);
 358             if (crm_is_daemon_name(value)) {
 359                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 360             }
 361         }
 362     }
 363 
 364     /* Allow cluster daemons more leeway before being evicted */
 365     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 366         const char *qmax = cib_config_lookup("cluster-ipc-limit");
 367 
 368         if (pcmk__set_client_queue_max(cib_client, qmax)) {
 369             crm_trace("IPC threshold for client %s[%u] is now %u",
 370                       pcmk__client_name(cib_client), cib_client->pid,
 371                       cib_client->queue_max);
 372         }
 373     }
 374 
 375     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
 376     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 377 
 378     CRM_LOG_ASSERT(cib_client->user != NULL);
 379     pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
 380 
 381     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 382     free_xml(op_request);
 383 
 384     return 0;
 385 }
 386 
 387 static uint64_t ping_seq = 0;
 388 static char *ping_digest = NULL;
 389 static bool ping_modified_since = FALSE;
 390 
 391 static gboolean
 392 cib_digester_cb(gpointer data)
     /*  */
 393 {
 394     if (based_is_primary) {
 395         char buffer[32];
 396         xmlNode *ping = create_xml_node(NULL, "ping");
 397 
 398         ping_seq++;
 399         free(ping_digest);
 400         ping_digest = NULL;
 401         ping_modified_since = FALSE;
 402         snprintf(buffer, 32, "%" PRIu64, ping_seq);
 403         crm_trace("Requesting peer digests (%s)", buffer);
 404 
 405         crm_xml_add(ping, F_TYPE, "cib");
 406         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
 407         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 408 
 409         crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 410         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 411 
 412         free_xml(ping);
 413     }
 414     return FALSE;
 415 }
 416 
 417 static void
 418 process_ping_reply(xmlNode *reply) 
     /*  */
 419 {
 420     uint64_t seq = 0;
 421     const char *host = crm_element_value(reply, F_ORIG);
 422 
 423     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
 424     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
 425     const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
 426 
 427     if (seq_s == NULL) {
 428         crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
 429         return;
 430 
 431     } else {
 432         long long seq_ll;
 433 
 434         if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
 435             return;
 436         }
 437         seq = (uint64_t) seq_ll;
 438     }
 439 
 440     if(digest == NULL) {
 441         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 442 
 443     } else if(seq != ping_seq) {
 444         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 445 
 446     } else if(ping_modified_since) {
 447         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 448 
 449     } else {
 450         const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
 451 
 452         if(ping_digest == NULL) {
 453             crm_trace("Calculating new digest");
 454             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
 455         }
 456 
 457         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 458         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 459             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
 460 
 461             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 462                        crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
 463                        crm_element_value(the_cib, XML_ATTR_GENERATION),
 464                        crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
 465                        ping_digest, host,
 466                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
 467                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
 468                        remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
 469                        digest, remote_cib);
 470 
 471             if(remote_cib && remote_cib->children) {
 472                 // Additional debug
 473                 xml_calculate_changes(the_cib, remote_cib);
 474                 pcmk__log_xml_changes(LOG_INFO, remote_cib);
 475                 crm_trace("End of differences");
 476             }
 477 
 478             free_xml(remote_cib);
 479             sync_our_cib(reply, FALSE);
 480         }
 481     }
 482 }
 483 
 484 static void
 485 local_notify_destroy_callback(gpointer data)
     /*  */
 486 {
 487     cib_local_notify_t *notify = data;
 488 
 489     free_xml(notify->notify_src);
 490     free(notify->client_id);
 491     free(notify);
 492 }
 493 
 494 static void
 495 check_local_notify(int bcast_id)
     /*  */
 496 {
 497     const cib_local_notify_t *notify = NULL;
 498 
 499     if (!local_notify_queue) {
 500         return;
 501     }
 502 
 503     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 504 
 505     if (notify) {
 506         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
 507                         notify->from_peer);
 508         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
 509     }
 510 }
 511 
 512 static void
 513 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
     /*  */
 514                    gboolean from_peer)
 515 {
 516     cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
 517 
 518     notify->notify_src = notify_src;
 519     notify->client_id = strdup(client_id);
 520     notify->sync_reply = sync_reply;
 521     notify->from_peer = from_peer;
 522 
 523     if (!local_notify_queue) {
 524         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
 525     }
 526     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
 527     // cppcheck doesn't know notify will get freed when hash table is destroyed
 528     // cppcheck-suppress memleak
 529 }
 530 
 531 static void
 532 parse_local_options_v1(const pcmk__client_t *cib_client,
     /*  */
 533                        const cib__operation_t *operation, int call_options,
 534                        const char *host, const char *op, gboolean *local_notify,
 535                        gboolean *needs_reply, gboolean *process,
 536                        gboolean *needs_forward)
 537 {
 538     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 539         && !pcmk_is_set(call_options, cib_inhibit_bcast)) {
 540         /* we need to send an update anyway */
 541         *needs_reply = TRUE;
 542     } else {
 543         *needs_reply = FALSE;
 544     }
 545 
 546     if (host == NULL && (call_options & cib_scope_local)) {
 547         crm_trace("Processing locally scoped %s op from client %s",
 548                   op, pcmk__client_name(cib_client));
 549         *local_notify = TRUE;
 550 
 551     } else if ((host == NULL) && based_is_primary) {
 552         crm_trace("Processing %s op locally from client %s as primary",
 553                   op, pcmk__client_name(cib_client));
 554         *local_notify = TRUE;
 555 
 556     } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 557         crm_trace("Processing locally addressed %s op from client %s",
 558                   op, pcmk__client_name(cib_client));
 559         *local_notify = TRUE;
 560 
 561     } else if (stand_alone) {
 562         *needs_forward = FALSE;
 563         *local_notify = TRUE;
 564         *process = TRUE;
 565 
 566     } else {
 567         crm_trace("%s op from %s needs to be forwarded to client %s",
 568                   op, pcmk__client_name(cib_client),
 569                   pcmk__s(host, "the primary instance"));
 570         *needs_forward = TRUE;
 571         *process = FALSE;
 572     }
 573 }
 574 
 575 static void
 576 parse_local_options_v2(const pcmk__client_t *cib_client,
     /*  */
 577                        const cib__operation_t *operation, int call_options,
 578                        const char *host, const char *op, gboolean *local_notify,
 579                        gboolean *needs_reply, gboolean *process,
 580                        gboolean *needs_forward)
 581 {
 582     // Process locally and notify local client
 583     *process = TRUE;
 584     *needs_reply = FALSE;
 585     *local_notify = TRUE;
 586     *needs_forward = FALSE;
 587 
 588     if (pcmk_is_set(operation->flags, cib__op_attr_local)) {
 589         /* Always process locally if cib__op_attr_local is set.
 590          *
 591          * @COMPAT: Currently host is ignored. At a compatibility break, throw
 592          * an error (from cib_process_request() or earlier) if host is not NULL or
 593          * OUR_NODENAME.
 594          */
 595         crm_trace("Processing always-local %s op from client %s",
 596                   op, pcmk__client_name(cib_client));
 597 
 598         if (!pcmk__str_eq(host, OUR_NODENAME,
 599                           pcmk__str_casei|pcmk__str_null_matches)) {
 600 
 601             crm_warn("Operation '%s' is always local but its target host is "
 602                      "set to '%s'",
 603                      op, host);
 604         }
 605         return;
 606     }
 607 
 608     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 609         || !pcmk__str_eq(host, OUR_NODENAME,
 610                          pcmk__str_casei|pcmk__str_null_matches)) {
 611 
 612         // Forward modifying and non-local requests via cluster
 613         *process = FALSE;
 614         *needs_reply = FALSE;
 615         *local_notify = FALSE;
 616         *needs_forward = TRUE;
 617 
 618         crm_trace("%s op from %s needs to be forwarded to %s",
 619                   op, pcmk__client_name(cib_client),
 620                   pcmk__s(host, "all nodes"));
 621         return;
 622     }
 623 
 624     if (stand_alone) {
 625         crm_trace("Processing %s op from client %s (stand-alone)",
 626                   op, pcmk__client_name(cib_client));
 627 
 628     } else {
 629         crm_trace("Processing %saddressed %s op from client %s",
 630                   ((host != NULL)? "locally " : "un"),
 631                   op, pcmk__client_name(cib_client));
 632     }
 633 }
 634 
 635 static void
 636 parse_local_options(const pcmk__client_t *cib_client,
     /*  */
 637                     const cib__operation_t *operation, int call_options,
 638                     const char *host, const char *op, gboolean *local_notify,
 639                     gboolean *needs_reply, gboolean *process,
 640                     gboolean *needs_forward)
 641 {
 642     if(cib_legacy_mode()) {
 643         parse_local_options_v1(cib_client, operation, call_options, host,
 644                                op, local_notify, needs_reply, process,
 645                                needs_forward);
 646     } else {
 647         parse_local_options_v2(cib_client, operation, call_options, host,
 648                                op, local_notify, needs_reply, process,
 649                                needs_forward);
 650     }
 651 }
 652 
 653 static gboolean
 654 parse_peer_options_v1(const cib__operation_t *operation, xmlNode *request,
     /*  */
 655                       gboolean *local_notify, gboolean *needs_reply,
 656                       gboolean *process)
 657 {
 658     const char *op = NULL;
 659     const char *host = NULL;
 660     const char *delegated = NULL;
 661     const char *originator = crm_element_value(request, F_ORIG);
 662     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 663 
 664     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 665 
 666     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 667         *needs_reply = FALSE;
 668         if (is_reply) {
 669             *local_notify = TRUE;
 670             crm_trace("Processing global/peer update from %s"
 671                       " that originated from us", originator);
 672         } else {
 673             crm_trace("Processing global/peer update from %s", originator);
 674         }
 675         return TRUE;
 676     }
 677 
 678     op = crm_element_value(request, F_CIB_OPERATION);
 679     crm_trace("Processing legacy %s request sent by %s", op, originator);
 680 
 681     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 682         /* Always process these */
 683         *local_notify = FALSE;
 684         if (reply_to == NULL || is_reply) {
 685             *process = TRUE;
 686         }
 687         if (is_reply) {
 688             *needs_reply = FALSE;
 689         }
 690         return *process;
 691     }
 692 
 693     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 694         process_ping_reply(request);
 695         return FALSE;
 696     }
 697 
 698     if (is_reply) {
 699         crm_trace("Forward reply sent from %s to local clients", originator);
 700         *process = FALSE;
 701         *needs_reply = FALSE;
 702         *local_notify = TRUE;
 703         return TRUE;
 704     }
 705 
 706     host = crm_element_value(request, F_CIB_HOST);
 707     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 708         crm_trace("Processing %s request sent to us from %s", op, originator);
 709         return TRUE;
 710 
 711     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 712         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
 713         *needs_reply = TRUE;
 714         return TRUE;
 715 
 716     } else if ((host == NULL) && based_is_primary) {
 717         crm_trace("Processing %s request sent to primary instance from %s",
 718                   op, originator);
 719         return TRUE;
 720     }
 721 
 722     delegated = crm_element_value(request, F_CIB_DELEGATED);
 723     if (delegated != NULL) {
 724         crm_trace("Ignoring message for primary instance");
 725 
 726     } else if (host != NULL) {
 727         /* this is for a specific instance and we're not it */
 728         crm_trace("Ignoring msg for instance on %s", host);
 729 
 730     } else if ((reply_to == NULL) && !based_is_primary) {
 731         // This is for the primary instance, and we're not it
 732         crm_trace("Ignoring reply for primary instance");
 733 
 734     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 735         if (reply_to != NULL) {
 736             crm_debug("Processing %s from %s", op, originator);
 737             *needs_reply = FALSE;
 738 
 739         } else {
 740             crm_debug("Processing %s reply from %s", op, originator);
 741         }
 742         return TRUE;
 743 
 744     } else {
 745         crm_err("Nothing for us to do?");
 746         crm_log_xml_err(request, "Peer[inbound]");
 747     }
 748 
 749     return FALSE;
 750 }
 751 
 752 static gboolean
 753 parse_peer_options_v2(const cib__operation_t *operation, xmlNode *request,
     /*  */
 754                       gboolean *local_notify, gboolean *needs_reply,
 755                       gboolean *process)
 756 {
 757     const char *host = NULL;
 758     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
 759     const char *op = crm_element_value(request, F_CIB_OPERATION);
 760     const char *originator = crm_element_value(request, F_ORIG);
 761     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 762 
 763     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 764 
 765     if (originator == NULL) { // Shouldn't be possible
 766         originator = "peer";
 767     }
 768 
 769     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
 770         /* sync_our_cib() sets F_CIB_ISREPLY */
 771         if (reply_to) {
 772             delegated = reply_to;
 773         }
 774         goto skip_is_reply;
 775 
 776     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
 777                             pcmk__str_none)) {
 778         // Nothing to do
 779 
 780     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 781         process_ping_reply(request);
 782         return FALSE;
 783 
 784     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
 785         /* Only the DC (node with the oldest software) should process
 786          * this operation if F_CIB_SCHEMA_MAX is unset
 787          *
 788          * If the DC is happy it will then send out another
 789          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
 790          * upgrade.
 791          *
 792          * Except this time F_CIB_SCHEMA_MAX will be set which puts a
 793          * limit on how far newer nodes will go
 794          */
 795         const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
 796         const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
 797 
 798         crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s",
 799                   (is_reply? "reply" : "request"),
 800                   (based_is_primary? "primary" : "secondary"),
 801                   pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none"));
 802 
 803         if (upgrade_rc != NULL) {
 804             // Our upgrade request was rejected by DC, notify clients of result
 805             crm_xml_add(request, F_CIB_RC, upgrade_rc);
 806 
 807         } else if ((max == NULL) && based_is_primary) {
 808             /* We are the DC, check if this upgrade is allowed */
 809             goto skip_is_reply;
 810 
 811         } else if(max) {
 812             /* Ok, go ahead and upgrade to 'max' */
 813             goto skip_is_reply;
 814 
 815         } else {
 816             // Ignore broadcast client requests when we're not primary
 817             return FALSE;
 818         }
 819 
 820     } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 821         crm_info("Detected legacy %s global update from %s", op, originator);
 822         send_sync_request(NULL);
 823         legacy_mode = TRUE;
 824         return FALSE;
 825 
 826     } else if (is_reply
 827                && pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
 828         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 829         return FALSE;
 830 
 831     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 832         *local_notify = FALSE;
 833         if (reply_to == NULL) {
 834             *process = TRUE;
 835         } else { // Not possible?
 836             crm_debug("Ignoring shutdown request from %s because reply_to=%s",
 837                       originator, reply_to);
 838         }
 839         return *process;
 840     }
 841 
 842     if (is_reply) {
 843         crm_trace("Will notify local clients for %s reply from %s",
 844                   op, originator);
 845         *process = FALSE;
 846         *needs_reply = FALSE;
 847         *local_notify = TRUE;
 848         return TRUE;
 849     }
 850 
 851   skip_is_reply:
 852     *process = TRUE;
 853     *needs_reply = FALSE;
 854 
 855     *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
 856 
 857     host = crm_element_value(request, F_CIB_HOST);
 858     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 859         crm_trace("Processing %s request sent to us from %s", op, originator);
 860         *needs_reply = TRUE;
 861         return TRUE;
 862 
 863     } else if (host != NULL) {
 864         crm_trace("Ignoring %s request intended for CIB manager on %s",
 865                   op, host);
 866         return FALSE;
 867 
 868     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 869         *needs_reply = TRUE;
 870     }
 871 
 872     crm_trace("Processing %s request broadcast by %s call %s on %s "
 873               "(local clients will%s be notified)", op,
 874               pcmk__s(crm_element_value(request, F_CIB_CLIENTNAME), "client"),
 875               pcmk__s(crm_element_value(request, F_CIB_CALLID), "without ID"),
 876               originator, (*local_notify? "" : "not"));
 877     return TRUE;
 878 }
 879 
 880 static gboolean
 881 parse_peer_options(const cib__operation_t *operation, xmlNode *request,
     /*  */
 882                    gboolean *local_notify, gboolean *needs_reply,
 883                    gboolean *process)
 884 {
 885     /* TODO: What happens when an update comes in after node A
 886      * requests the CIB from node B, but before it gets the reply (and
 887      * sends out the replace operation)
 888      */
 889     if(cib_legacy_mode()) {
 890         return parse_peer_options_v1(operation, request, local_notify,
 891                                      needs_reply, process);
 892     } else {
 893         return parse_peer_options_v2(operation, request, local_notify,
 894                                      needs_reply, process);
 895     }
 896 }
 897 
 898 /*!
 899  * \internal
 900  * \brief Forward a CIB request to the appropriate target host(s)
 901  *
 902  * \param[in] request  CIB request to forward
 903  */
 904 static void
 905 forward_request(xmlNode *request)
     /*  */
 906 {
 907     const char *op = crm_element_value(request, F_CIB_OPERATION);
 908     const char *section = crm_element_value(request, F_CIB_SECTION);
 909     const char *host = crm_element_value(request, F_CIB_HOST);
 910     const char *originator = crm_element_value(request, F_ORIG);
 911     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
 912     const char *call_id = crm_element_value(request, F_CIB_CALLID);
 913 
 914     int log_level = LOG_INFO;
 915 
 916     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
 917         log_level = LOG_DEBUG;
 918     }
 919 
 920     do_crm_log(log_level,
 921                "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
 922                pcmk__s(op, "invalid"),
 923                pcmk__s(section, "all"),
 924                pcmk__s(host, (cib_legacy_mode()? "primary" : "all")),
 925                pcmk__s(originator, "local"),
 926                pcmk__s(client_name, "unspecified"),
 927                pcmk__s(call_id, "unspecified"));
 928 
 929     crm_xml_add(request, F_CIB_DELEGATED, OUR_NODENAME);
 930 
 931     send_cluster_message(((host != NULL)? crm_get_peer(0, host) : NULL),
 932                          crm_msg_cib, request, FALSE);
 933 
 934     // Return the request to its original state
 935     xml_remove_prop(request, F_CIB_DELEGATED);
 936 }
 937 
 938 static gboolean
 939 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
     /*  */
 940 {
 941     CRM_ASSERT(msg != NULL);
 942 
 943     if (broadcast) {
 944         /* @COMPAT: Legacy code
 945          *
 946          * This successful call modified the CIB, and the change needs to be
 947          * broadcast (sent via cluster to all nodes).
 948          */
 949         int diff_add_updates = 0;
 950         int diff_add_epoch = 0;
 951         int diff_add_admin_epoch = 0;
 952 
 953         int diff_del_updates = 0;
 954         int diff_del_epoch = 0;
 955         int diff_del_admin_epoch = 0;
 956 
 957         const char *digest = NULL;
 958         int format = 1;
 959 
 960         CRM_LOG_ASSERT(result_diff != NULL);
 961         digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
 962         crm_element_value_int(result_diff, PCMK_XA_FORMAT, &format);
 963 
 964         cib_diff_version_details(result_diff,
 965                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
 966                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 967 
 968         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
 969                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
 970                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 971 
 972         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 973         pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
 974         crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
 975         crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
 976 
 977         if (format == 1) {
 978             CRM_ASSERT(digest != NULL);
 979         }
 980 
 981         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
 982         crm_log_xml_explicit(msg, "copy");
 983         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 984 
 985     } else if (originator != NULL) {
 986         /* send reply via HA to originating node */
 987         crm_trace("Sending request result to %s only", originator);
 988         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 989         return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
 990     }
 991 
 992     return FALSE;
 993 }
 994 
 995 /*!
 996  * \internal
 997  * \brief Handle an IPC or CPG message containing a request
 998  *
 999  * \param[in,out] request        Request XML
1000  * \param[in] privileged         Whether privileged commands may be run
1001  *                               (see cib_server_ops[] definition)
1002  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
1003  *
1004  * \return Legacy Pacemaker return code
1005  */
1006 int
1007 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
1008                     const pcmk__client_t *cib_client)
1009 {
1010     // @TODO: Break into multiple smaller functions
1011     int call_options = 0;
1012 
1013     gboolean process = TRUE;        // Whether to process request locally now
1014     gboolean is_update = TRUE;      // Whether request would modify CIB
1015     gboolean needs_reply = TRUE;    // Whether to build a reply
1016     gboolean local_notify = FALSE;  // Whether to notify (local) requester
1017     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
1018 
1019     xmlNode *op_reply = NULL;
1020     xmlNode *result_diff = NULL;
1021 
1022     int rc = pcmk_ok;
1023     const char *op = crm_element_value(request, F_CIB_OPERATION);
1024     const char *originator = crm_element_value(request, F_ORIG);
1025     const char *host = crm_element_value(request, F_CIB_HOST);
1026     const char *target = NULL;
1027     const char *call_id = crm_element_value(request, F_CIB_CALLID);
1028     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
1029     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
1030     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
1031 
1032     const cib__operation_t *operation = NULL;
1033     cib__op_fn_t op_function = NULL;
1034 
1035     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
1036 
1037     if ((host != NULL) && (*host == '\0')) {
1038         host = NULL;
1039     }
1040 
1041     // @TODO: Improve trace messages. Target is accurate only for legacy mode.
1042     if (host) {
1043         target = host;
1044 
1045     } else if (call_options & cib_scope_local) {
1046         target = "local host";
1047 
1048     } else {
1049         target = "primary";
1050     }
1051 
1052     if (cib_client == NULL) {
1053         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
1054                   op, client_name, call_id, originator, target, reply_to);
1055     } else {
1056         crm_xml_add(request, F_ORIG, OUR_NODENAME);
1057         crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
1058     }
1059 
1060     rc = cib__get_operation(op, &operation);
1061     rc = pcmk_rc2legacy(rc);
1062     if (rc != pcmk_ok) {
1063         /* TODO: construct error reply? */
1064         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
1065         return rc;
1066     }
1067 
1068     op_function = based_get_op_function(operation);
1069     if (op_function == NULL) {
1070         crm_err("Operation %s not supported by CIB manager", op);
1071         return -EOPNOTSUPP;
1072     }
1073 
1074     if (cib_client != NULL) {
1075         parse_local_options(cib_client, operation, call_options, host, op,
1076                             &local_notify, &needs_reply, &process,
1077                             &needs_forward);
1078 
1079     } else if (!parse_peer_options(operation, request, &local_notify,
1080                                    &needs_reply, &process)) {
1081         return rc;
1082     }
1083 
1084     if (pcmk_is_set(call_options, cib_transaction)) {
1085         /* All requests in a transaction are processed locally against a working
1086          * CIB copy, and we don't notify for individual requests because the
1087          * entire transaction is atomic.
1088          *
1089          * We still call the option parser functions above, for the sake of log
1090          * messages and checking whether we're the target for peer requests.
1091          */
1092         process = TRUE;
1093         needs_reply = FALSE;
1094         local_notify = FALSE;
1095         needs_forward = FALSE;
1096     }
1097 
1098     is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies);
1099 
1100     if (pcmk_is_set(call_options, cib_discard_reply)) {
1101         /* If the request will modify the CIB, and we are in legacy mode, we
1102          * need to build a reply so we can broadcast a diff, even if the
1103          * requester doesn't want one.
1104          */
1105         needs_reply = is_update && cib_legacy_mode();
1106         local_notify = FALSE;
1107         crm_trace("Client is not interested in the reply");
1108     }
1109 
1110     if (needs_forward) {
1111         forward_request(request);
1112         return rc;
1113     }
1114 
1115     if (cib_status != pcmk_ok) {
1116         rc = cib_status;
1117         crm_err("Operation ignored, cluster configuration is invalid."
1118                 " Please repair and restart: %s", pcmk_strerror(cib_status));
1119 
1120         op_reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1121                                     the_cib);
1122 
1123     } else if (process) {
1124         time_t finished = 0;
1125         time_t now = time(NULL);
1126         int level = LOG_INFO;
1127         const char *section = crm_element_value(request, F_CIB_SECTION);
1128 
1129         rc = cib_process_command(request, operation, op_function, &op_reply,
1130                                  &result_diff, privileged);
1131 
1132         if (!is_update) {
1133             level = LOG_TRACE;
1134 
1135         } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1136             switch (rc) {
1137                 case pcmk_ok:
1138                     level = LOG_INFO;
1139                     break;
1140                 case -pcmk_err_old_data:
1141                 case -pcmk_err_diff_resync:
1142                 case -pcmk_err_diff_failed:
1143                     level = LOG_TRACE;
1144                     break;
1145                 default:
1146                     level = LOG_ERR;
1147             }
1148 
1149         } else if (rc != pcmk_ok) {
1150             level = LOG_WARNING;
1151         }
1152 
1153         do_crm_log(level,
1154                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
1155                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
1156                    originator ? originator : "local", client_name, call_id,
1157                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
1158                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
1159                    the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
1160 
1161         finished = time(NULL);
1162         if ((finished - now) > 3) {
1163             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
1164             crm_write_blackbox(0, NULL);
1165         }
1166 
1167         if (op_reply == NULL && (needs_reply || local_notify)) {
1168             crm_err("Unexpected NULL reply to message");
1169             crm_log_xml_err(request, "null reply");
1170             needs_reply = FALSE;
1171             local_notify = FALSE;
1172         }
1173     }
1174 
1175     if (is_update && !cib_legacy_mode()) {
1176         crm_trace("Completed pre-sync update from %s/%s/%s%s",
1177                   originator ? originator : "local", client_name, call_id,
1178                   local_notify?" with local notification":"");
1179 
1180     } else if (!needs_reply || stand_alone) {
1181         // This was a non-originating secondary update
1182         crm_trace("Completed update as secondary");
1183 
1184     } else if (cib_legacy_mode() &&
1185                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
1186         gboolean broadcast = FALSE;
1187 
1188         cib_local_bcast_num++;
1189         crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
1190         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
1191 
1192         if (broadcast && client_id && local_notify && op_reply) {
1193 
1194             /* If we have been asked to sync the reply,
1195              * and a bcast msg has gone out, we queue the local notify
1196              * until we know the bcast message has been received */
1197             local_notify = FALSE;
1198             crm_trace("Queuing local %ssync notification for %s",
1199                       (call_options & cib_sync_call) ? "" : "a-", client_id);
1200 
1201             queue_local_notify(op_reply, client_id,
1202                                pcmk_is_set(call_options, cib_sync_call),
1203                                (cib_client == NULL));
1204             op_reply = NULL;    /* the reply is queued, so don't free here */
1205         }
1206 
1207     } else if ((cib_client == NULL)
1208                && !pcmk_is_set(call_options, cib_discard_reply)) {
1209 
1210         if (is_update == FALSE || result_diff == NULL) {
1211             crm_trace("Request not broadcast: R/O call");
1212 
1213         } else if (call_options & cib_inhibit_bcast) {
1214             crm_trace("Request not broadcast: inhibited");
1215 
1216         } else if (rc != pcmk_ok) {
1217             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
1218 
1219         } else {
1220             crm_trace("Directing reply to %s", originator);
1221         }
1222 
1223         send_peer_reply(op_reply, result_diff, originator, FALSE);
1224     }
1225 
1226     if (local_notify && client_id) {
1227         crm_trace("Performing local %ssync notification for %s",
1228                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
1229                   client_id);
1230         if (process == FALSE) {
1231             do_local_notify(request, client_id,
1232                             pcmk_is_set(call_options, cib_sync_call),
1233                             (cib_client == NULL));
1234         } else {
1235             do_local_notify(op_reply, client_id,
1236                             pcmk_is_set(call_options, cib_sync_call),
1237                             (cib_client == NULL));
1238         }
1239     }
1240 
1241     free_xml(op_reply);
1242     free_xml(result_diff);
1243 
1244     return rc;
1245 }
1246 
1247 /*!
1248  * \internal
1249  * \brief Get a CIB operation's input from the request XML
1250  *
1251  * \param[in]  request  CIB request XML
1252  * \param[in]  type     CIB operation type
1253  * \param[out] section  Where to store CIB section name
1254  *
1255  * \return Input XML for CIB operation
1256  *
1257  * \note If not \c NULL, the return value is a non-const pointer to part of
1258  *       \p request. The caller should not free it directly.
1259  */
1260 static xmlNode *
1261 prepare_input(const xmlNode *request, enum cib__op_type type,
     /*  */
1262               const char **section)
1263 {
1264     xmlNode *input = NULL;
1265 
1266     *section = NULL;
1267 
1268     switch (type) {
1269         case cib__op_apply_patch:
1270             if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1271                 input = get_message_xml(request, F_CIB_UPDATE_DIFF);
1272             } else {
1273                 input = get_message_xml(request, F_CIB_CALLDATA);
1274             }
1275             break;
1276 
1277         default:
1278             input = get_message_xml(request, F_CIB_CALLDATA);
1279             *section = crm_element_value(request, F_CIB_SECTION);
1280             break;
1281     }
1282 
1283     // Grab the specified section
1284     if ((*section != NULL) && pcmk__xe_is(input, XML_TAG_CIB)) {
1285         input = pcmk_find_cib_element(input, *section);
1286     }
1287 
1288     return input;
1289 }
1290 
1291 // v1 and v2 patch formats
1292 #define XPATH_CONFIG_CHANGE             \
1293     "//" XML_CIB_TAG_CRMCONFIG " | "    \
1294     "//" XML_DIFF_CHANGE                \
1295     "[contains(@" XML_DIFF_PATH ",'/" XML_CIB_TAG_CRMCONFIG "/')]"
1296 
1297 static bool
1298 contains_config_change(xmlNode *diff)
     /*  */
1299 {
1300     bool changed = false;
1301 
1302     if (diff) {
1303         xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
1304 
1305         if (numXpathResults(xpathObj) > 0) {
1306             changed = true;
1307         }
1308         freeXpathObject(xpathObj);
1309     }
1310     return changed;
1311 }
1312 
1313 static int
1314 cib_process_command(xmlNode *request, const cib__operation_t *operation,
     /*  */
1315                     cib__op_fn_t op_function, xmlNode **reply,
1316                     xmlNode **cib_diff, bool privileged)
1317 {
1318     xmlNode *input = NULL;
1319     xmlNode *output = NULL;
1320     xmlNode *result_cib = NULL;
1321 
1322     int call_options = 0;
1323 
1324     const char *op = NULL;
1325     const char *section = NULL;
1326     const char *call_id = crm_element_value(request, F_CIB_CALLID);
1327     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
1328     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
1329     const char *originator = crm_element_value(request, F_ORIG);
1330 
1331     int rc = pcmk_ok;
1332 
1333     bool config_changed = false;
1334     bool manage_counters = true;
1335 
1336     static mainloop_timer_t *digest_timer = NULL;
1337 
1338     CRM_ASSERT(cib_status == pcmk_ok);
1339 
1340     if(digest_timer == NULL) {
1341         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1342     }
1343 
1344     *reply = NULL;
1345     *cib_diff = NULL;
1346 
1347     /* Start processing the request... */
1348     op = crm_element_value(request, F_CIB_OPERATION);
1349     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
1350 
1351     if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) {
1352         rc = -EACCES;
1353         crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc));
1354         goto done;
1355     }
1356 
1357     input = prepare_input(request, operation->type, &section);
1358 
1359     if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
1360         rc = cib_perform_op(op, call_options, op_function, true, section,
1361                             request, input, false, &config_changed, &the_cib,
1362                             &result_cib, NULL, &output);
1363 
1364         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
1365         goto done;
1366     }
1367 
1368     /* @COMPAT: Handle a valid write action (legacy)
1369      *
1370      * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff
1371      * portion is. However, F_CIB_GLOBAL_UPDATE may be set by a sync operation
1372      * even in non-legacy mode, and manage_counters tells xml_create_patchset()
1373      * whether to update version/epoch info.
1374      */
1375     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1376         manage_counters = false;
1377         cib__set_call_options(call_options, "call", cib_force_diff);
1378         crm_trace("Global update detected");
1379 
1380         CRM_LOG_ASSERT(pcmk__str_any_of(op,
1381                                         PCMK__CIB_REQUEST_APPLY_PATCH,
1382                                         PCMK__CIB_REQUEST_REPLACE,
1383                                         NULL));
1384     }
1385 
1386     ping_modified_since = TRUE;
1387     if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
1388         crm_trace("Skipping update: inhibit broadcast");
1389         manage_counters = false;
1390     }
1391 
1392     // result_cib must not be modified after cib_perform_op() returns
1393     rc = cib_perform_op(op, call_options, op_function, false, section,
1394                         request, input, manage_counters, &config_changed,
1395                         &the_cib, &result_cib, cib_diff, &output);
1396 
1397     // @COMPAT: Legacy code
1398     if (!manage_counters) {
1399         int format = 1;
1400 
1401         // If the diff is NULL at this point, it's because nothing changed
1402         if (*cib_diff != NULL) {
1403             crm_element_value_int(*cib_diff, PCMK_XA_FORMAT, &format);
1404         }
1405 
1406         if (format == 1) {
1407             config_changed = cib__config_changed_v1(NULL, NULL, cib_diff);
1408         }
1409     }
1410 
1411     /* Always write to disk for successful ops with the flag set. This also
1412      * negates the need to detect ordering changes.
1413      */
1414     if ((rc == pcmk_ok)
1415         && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1416 
1417         config_changed = true;
1418     }
1419 
1420     if ((rc == pcmk_ok)
1421         && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) {
1422 
1423         if (result_cib != the_cib) {
1424             if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1425                 config_changed = true;
1426             }
1427 
1428             crm_trace("Activating %s->%s%s",
1429                       crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
1430                       crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
1431                       (config_changed? " changed" : ""));
1432 
1433             rc = activateCibXml(result_cib, config_changed, op);
1434             if (rc != pcmk_ok) {
1435                 crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc));
1436             }
1437         }
1438 
1439         if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
1440             cib_read_config(config_hash, result_cib);
1441         }
1442 
1443         /* @COMPAT Nodes older than feature set 3.19.0 don't support
1444          * transactions. In a mixed-version cluster with nodes <3.19.0, we must
1445          * sync the updated CIB, so that the older nodes receive the changes.
1446          * Any node that has already applied the transaction will ignore the
1447          * synced CIB.
1448          *
1449          * To ensure the updated CIB is synced from only one node, we sync it
1450          * from the originator.
1451          */
1452         if ((operation->type == cib__op_commit_transact)
1453             && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei)
1454             && compare_version(crm_element_value(the_cib, XML_ATTR_CRM_VERSION),
1455                                "3.19.0") < 0) {
1456 
1457             sync_our_cib(request, TRUE);
1458         }
1459 
1460         mainloop_timer_stop(digest_timer);
1461         mainloop_timer_start(digest_timer);
1462 
1463     } else if (rc == -pcmk_err_schema_validation) {
1464         CRM_ASSERT(result_cib != the_cib);
1465 
1466         if (output != NULL) {
1467             crm_log_xml_info(output, "cib:output");
1468             free_xml(output);
1469         }
1470 
1471         output = result_cib;
1472 
1473     } else {
1474         crm_trace("Not activating %d %d %s", rc,
1475                   pcmk_is_set(call_options, cib_dryrun),
1476                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
1477 
1478         if (result_cib != the_cib) {
1479             free_xml(result_cib);
1480         }
1481     }
1482 
1483     if (!pcmk_any_flags_set(call_options,
1484                             cib_dryrun|cib_inhibit_notify|cib_transaction)) {
1485         crm_trace("Sending notifications %d",
1486                   pcmk_is_set(call_options, cib_dryrun));
1487         cib_diff_notify(op, rc, call_id, client_id, client_name, originator,
1488                         input, *cib_diff);
1489     }
1490 
1491     pcmk__log_xml_patchset(LOG_TRACE, *cib_diff);
1492 
1493   done:
1494     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
1495         *reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1496                                   output);
1497     }
1498 
1499     if (output != the_cib) {
1500         free_xml(output);
1501     }
1502     crm_trace("done");
1503     return rc;
1504 }
1505 
1506 void
1507 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1508 {
1509     const char *reason = NULL;
1510     const char *originator = crm_element_value(msg, F_ORIG);
1511 
1512     if (cib_legacy_mode()
1513         && pcmk__str_eq(originator, OUR_NODENAME,
1514                         pcmk__str_casei|pcmk__str_null_matches)) {
1515         /* message is from ourselves */
1516         int bcast_id = 0;
1517 
1518         if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
1519             check_local_notify(bcast_id);
1520         }
1521         return;
1522 
1523     } else if (crm_peer_cache == NULL) {
1524         reason = "membership not established";
1525         goto bail;
1526     }
1527 
1528     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
1529         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
1530     }
1531 
1532     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1533     cib_process_request(msg, TRUE, NULL);
1534     return;
1535 
1536   bail:
1537     if (reason) {
1538         const char *seq = crm_element_value(msg, F_SEQ);
1539         const char *op = crm_element_value(msg, F_CIB_OPERATION);
1540 
1541         crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
1542     }
1543 }
1544 
1545 static gboolean
1546 cib_force_exit(gpointer data)
     /*  */
1547 {
1548     crm_notice("Forcing exit!");
1549     terminate_cib(__func__, CRM_EX_ERROR);
1550     return FALSE;
1551 }
1552 
1553 static void
1554 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1555 {
1556     pcmk__client_t *a_client = value;
1557 
1558     crm_err("Can't disconnect client %s: Not implemented",
1559             pcmk__client_name(a_client));
1560 }
1561 
1562 static void
1563 initiate_exit(void)
     /*  */
1564 {
1565     int active = 0;
1566     xmlNode *leaving = NULL;
1567 
1568     active = crm_active_peers();
1569     if (active < 2) { // This is the last active node
1570         terminate_cib(__func__, 0);
1571         return;
1572     }
1573 
1574     crm_info("Sending shutdown request to %d peers", active);
1575 
1576     leaving = create_xml_node(NULL, "exit-notification");
1577     crm_xml_add(leaving, F_TYPE, "cib");
1578     crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);
1579 
1580     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
1581     free_xml(leaving);
1582 
1583     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1584 }
1585 
1586 void
1587 cib_shutdown(int nsig)
     /*  */
1588 {
1589     struct qb_ipcs_stats srv_stats;
1590 
1591     if (cib_shutdown_flag == FALSE) {
1592         int disconnects = 0;
1593         qb_ipcs_connection_t *c = NULL;
1594 
1595         cib_shutdown_flag = TRUE;
1596 
1597         c = qb_ipcs_connection_first_get(ipcs_rw);
1598         while (c != NULL) {
1599             qb_ipcs_connection_t *last = c;
1600 
1601             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1602 
1603             crm_debug("Disconnecting r/w client %p...", last);
1604             qb_ipcs_disconnect(last);
1605             qb_ipcs_connection_unref(last);
1606             disconnects++;
1607         }
1608 
1609         c = qb_ipcs_connection_first_get(ipcs_ro);
1610         while (c != NULL) {
1611             qb_ipcs_connection_t *last = c;
1612 
1613             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1614 
1615             crm_debug("Disconnecting r/o client %p...", last);
1616             qb_ipcs_disconnect(last);
1617             qb_ipcs_connection_unref(last);
1618             disconnects++;
1619         }
1620 
1621         c = qb_ipcs_connection_first_get(ipcs_shm);
1622         while (c != NULL) {
1623             qb_ipcs_connection_t *last = c;
1624 
1625             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1626 
1627             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1628             qb_ipcs_disconnect(last);
1629             qb_ipcs_connection_unref(last);
1630             disconnects++;
1631         }
1632 
1633         disconnects += pcmk__ipc_client_count();
1634 
1635         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1636         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1637         crm_info("Disconnected %d clients", disconnects);
1638     }
1639 
1640     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1641 
1642     if (pcmk__ipc_client_count() == 0) {
1643         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1644         initiate_exit();
1645 
1646     } else {
1647         crm_info("Waiting on %d clients to disconnect (%d)",
1648                  pcmk__ipc_client_count(), srv_stats.active_connections);
1649     }
1650 }
1651 
1652 extern int remote_fd;
1653 extern int remote_tls_fd;
1654 
1655 /*!
1656  * \internal
1657  * \brief Close remote sockets, free the global CIB and quit
1658  *
1659  * \param[in] caller           Name of calling function (for log message)
1660  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1661  */
1662 void
1663 terminate_cib(const char *caller, int fast)
     /*  */
1664 {
1665     crm_info("%s: Exiting%s...", caller,
1666              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1667 
1668     if (remote_fd > 0) {
1669         close(remote_fd);
1670         remote_fd = 0;
1671     }
1672     if (remote_tls_fd > 0) {
1673         close(remote_tls_fd);
1674         remote_tls_fd = 0;
1675     }
1676 
1677     uninitializeCib();
1678 
1679     if (fast > 0) {
1680         /* Quit fast on error */
1681         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1682         crm_exit(fast);
1683 
1684     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1685         /* Quit via returning from the main loop. If fast == -1, we skip the
1686          * disconnect here, and it will be done when the main loop returns
1687          * (this allows the peer status callback to avoid messing with the
1688          * peer caches).
1689          */
1690         if (fast == 0) {
1691             crm_cluster_disconnect(crm_cluster);
1692         }
1693         g_main_loop_quit(mainloop);
1694 
1695     } else {
1696         /* Quit via clean exit. Even the peer status callback can disconnect
1697          * here, because we're not returning control to the caller. */
1698         crm_cluster_disconnect(crm_cluster);
1699         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1700         crm_exit(CRM_EX_OK);
1701     }
1702 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
