daemons/based/based

/* */
This source file includes following definitions.
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
create_cib_reply
do_local_notify
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
parse_local_options
parse_peer_options
forward_request
send_peer_reply
cib_process_request
prepare_input
contains_config_change
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
initiate_exit
cib_shutdown
terminate_cib
   1 /*
   2  * Copyright 2004-2025 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>   // PRIu64
  22 
  23 #include <glib.h>
  24 #include <libxml/tree.h>
  25 #include <libxml/xpath.h>               // xmlXPathObject, etc.
  26 
  27 #include <crm/crm.h>
  28 #include <crm/cib.h>
  29 #include <crm/cluster/internal.h>
  30 
  31 #include <crm/common/xml.h>
  32 #include <crm/common/remote_internal.h>
  33 
  34 #include <pacemaker-based.h>
  35 
  36 #define EXIT_ESCALATION_MS 10000
  37 
  38 qb_ipcs_service_t *ipcs_ro = NULL;
  39 qb_ipcs_service_t *ipcs_rw = NULL;
  40 qb_ipcs_service_t *ipcs_shm = NULL;
  41 
  42 static int cib_process_command(xmlNode *request,
  43                                const cib__operation_t *operation,
  44                                cib__op_fn_t op_function, xmlNode **reply,
  45                                xmlNode **cib_diff, bool privileged);
  46 
  47 static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
  48                                     size_t size, gboolean privileged);
  49 
  50 static int32_t
  51 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  52 {
  53     if (cib_shutdown_flag) {
  54         crm_info("Ignoring new IPC client [%d] during shutdown",
  55                  pcmk__client_pid(c));
  56         return -ECONNREFUSED;
  57     }
  58 
  59     if (pcmk__new_client(c, uid, gid) == NULL) {
  60         return -ENOMEM;
  61     }
  62     return 0;
  63 }
  64 
  65 static int32_t
  66 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  67 {
  68     pcmk__client_t *client = pcmk__find_client(c);
  69 
  70     crm_trace("%p message from %s", c, client->id);
  71     return cib_common_callback(c, data, size, TRUE);
  72 }
  73 
  74 static int32_t
  75 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  76 {
  77     pcmk__client_t *client = pcmk__find_client(c);
  78 
  79     crm_trace("%p message from %s", c, client->id);
  80     return cib_common_callback(c, data, size, FALSE);
  81 }
  82 
  83 /* Error code means? */
  84 static int32_t
  85 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
  86 {
  87     pcmk__client_t *client = pcmk__find_client(c);
  88 
  89     if (client == NULL) {
  90         return 0;
  91     }
  92     crm_trace("Connection %p", c);
  93     pcmk__free_client(client);
  94     return 0;
  95 }
  96 
  97 static void
  98 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
  99 {
 100     crm_trace("Connection %p", c);
 101     cib_ipc_closed(c);
 102     if (cib_shutdown_flag) {
 103         cib_shutdown(0);
 104     }
 105 }
 106 
 107 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 108     .connection_accept = cib_ipc_accept,
 109     .connection_created = NULL,
 110     .msg_process = cib_ipc_dispatch_ro,
 111     .connection_closed = cib_ipc_closed,
 112     .connection_destroyed = cib_ipc_destroy
 113 };
 114 
 115 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 116     .connection_accept = cib_ipc_accept,
 117     .connection_created = NULL,
 118     .msg_process = cib_ipc_dispatch_rw,
 119     .connection_closed = cib_ipc_closed,
 120     .connection_destroyed = cib_ipc_destroy
 121 };
 122 
 123 /*!
 124  * \internal
 125  * \brief Create reply XML for a CIB request
 126  *
 127  * \param[in] op            CIB operation type
 128  * \param[in] call_id       CIB call ID
 129  * \param[in] client_id     CIB client ID
 130  * \param[in] call_options  Group of <tt>enum cib_call_options</tt> flags
 131  * \param[in] rc            Request return code
 132  * \param[in] call_data     Request output data
 133  *
 134  * \return Reply XML (guaranteed not to be \c NULL)
 135  *
 136  * \note The caller is responsible for freeing the return value using
 137  *       \p pcmk__xml_free().
 138  */
 139 static xmlNode *
 140 create_cib_reply(const char *op, const char *call_id, const char *client_id,
     /*  */
 141                  uint32_t call_options, int rc, xmlNode *call_data)
 142 {
 143     xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_CIB_REPLY);
 144 
 145     crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_CIB);
 146     crm_xml_add(reply, PCMK__XA_CIB_OP, op);
 147     crm_xml_add(reply, PCMK__XA_CIB_CALLID, call_id);
 148     crm_xml_add(reply, PCMK__XA_CIB_CLIENTID, client_id);
 149     crm_xml_add_int(reply, PCMK__XA_CIB_CALLOPT, call_options);
 150     crm_xml_add_int(reply, PCMK__XA_CIB_RC, rc);
 151 
 152     if (call_data != NULL) {
 153         xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_CIB_CALLDATA);
 154 
 155         crm_trace("Attaching reply output");
 156         pcmk__xml_copy(wrapper, call_data);
 157     }
 158 
 159     crm_log_xml_explicit(reply, "cib:reply");
 160     return reply;
 161 }
 162 
 163 static void
 164 do_local_notify(const xmlNode *notify_src, const char *client_id,
     /*  */
 165                 bool sync_reply, bool from_peer)
 166 {
 167     int msg_id = 0;
 168     int rc = pcmk_rc_ok;
 169     pcmk__client_t *client_obj = NULL;
 170     uint32_t flags = crm_ipc_server_event;
 171 
 172     CRM_CHECK((notify_src != NULL) && (client_id != NULL), return);
 173 
 174     crm_element_value_int(notify_src, PCMK__XA_CIB_CALLID, &msg_id);
 175 
 176     client_obj = pcmk__find_client_by_id(client_id);
 177     if (client_obj == NULL) {
 178         crm_debug("Could not notify client %s%s %s of call %d result: "
 179                   "client no longer exists", client_id,
 180                   (from_peer? " (originator of delegated request)" : ""),
 181                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 182         return;
 183     }
 184 
 185     if (sync_reply) {
 186         flags = crm_ipc_flags_none;
 187         if (client_obj->ipcs != NULL) {
 188             msg_id = client_obj->request_id;
 189             client_obj->request_id = 0;
 190         }
 191     }
 192 
 193     switch (PCMK__CLIENT_TYPE(client_obj)) {
 194         case pcmk__client_ipc:
 195             rc = pcmk__ipc_send_xml(client_obj, msg_id, notify_src, flags);
 196             break;
 197         case pcmk__client_tls:
 198         case pcmk__client_tcp:
 199             rc = pcmk__remote_send_xml(client_obj->remote, notify_src);
 200             break;
 201         default:
 202             rc = EPROTONOSUPPORT;
 203             break;
 204     }
 205     if (rc == pcmk_rc_ok) {
 206         crm_trace("Notified %s client %s%s %s of call %d result",
 207                   pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 208                   pcmk__client_name(client_obj),
 209                   (from_peer? " (originator of delegated request)" : ""),
 210                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 211     } else {
 212         crm_warn("Could not notify %s client %s%s %s of call %d result: %s",
 213                  pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 214                  pcmk__client_name(client_obj),
 215                  (from_peer? " (originator of delegated request)" : ""),
 216                  (sync_reply? "synchronously" : "asynchronously"), msg_id,
 217                  pcmk_rc_str(rc));
 218     }
 219 }
 220 
 221 void
 222 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 223                            pcmk__client_t *cib_client, gboolean privileged)
 224 {
 225     const char *op = crm_element_value(op_request, PCMK__XA_CIB_OP);
 226     uint32_t call_options = cib_none;
 227     int rc = pcmk_rc_ok;
 228 
 229     rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 230                             cib_none);
 231     if (rc != pcmk_rc_ok) {
 232         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
 233     }
 234 
 235     /* Requests with cib_transaction set should not be sent to based directly
 236      * (outside of a commit-transaction request)
 237      */
 238     if (pcmk_is_set(call_options, cib_transaction)) {
 239         return;
 240     }
 241 
 242     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 243         if (flags & crm_ipc_client_response) {
 244             xmlNode *ack = pcmk__xe_create(NULL, __func__);
 245 
 246             crm_xml_add(ack, PCMK__XA_CIB_OP, CRM_OP_REGISTER);
 247             crm_xml_add(ack, PCMK__XA_CIB_CLIENTID, cib_client->id);
 248             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 249             cib_client->request_id = 0;
 250             pcmk__xml_free(ack);
 251         }
 252         return;
 253 
 254     } else if (pcmk__str_eq(op, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) {
 255         /* Update the notify filters for this client */
 256         int on_off = 0;
 257         crm_exit_t status = CRM_EX_OK;
 258         uint64_t bit = UINT64_C(0);
 259         const char *type = crm_element_value(op_request,
 260                                              PCMK__XA_CIB_NOTIFY_TYPE);
 261 
 262         crm_element_value_int(op_request, PCMK__XA_CIB_NOTIFY_ACTIVATE,
 263                               &on_off);
 264 
 265         crm_debug("Setting %s callbacks %s for client %s",
 266                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 267 
 268         if (pcmk__str_eq(type, PCMK__VALUE_CIB_POST_NOTIFY, pcmk__str_none)) {
 269             bit = cib_notify_post;
 270 
 271         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_PRE_NOTIFY,
 272                                 pcmk__str_none)) {
 273             bit = cib_notify_pre;
 274 
 275         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_UPDATE_CONFIRMATION,
 276                                 pcmk__str_none)) {
 277             bit = cib_notify_confirm;
 278 
 279         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_DIFF_NOTIFY,
 280                                 pcmk__str_none)) {
 281             bit = cib_notify_diff;
 282 
 283         } else {
 284             status = CRM_EX_INVALID_PARAM;
 285         }
 286 
 287         if (bit != 0) {
 288             if (on_off) {
 289                 pcmk__set_client_flags(cib_client, bit);
 290             } else {
 291                 pcmk__clear_client_flags(cib_client, bit);
 292             }
 293         }
 294 
 295         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status);
 296         return;
 297     }
 298 
 299     cib_process_request(op_request, privileged, cib_client);
 300 }
 301 
 302 int32_t
 303 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 304 {
 305     int rc = pcmk_rc_ok;
 306     uint32_t id = 0;
 307     uint32_t flags = 0;
 308     uint32_t call_options = cib_none;
 309     pcmk__client_t *cib_client = pcmk__find_client(c);
 310     xmlNode *op_request = NULL;
 311 
 312     if (cib_client == NULL) {
 313         crm_trace("Invalid client %p", c);
 314         return 0;
 315     }
 316 
 317     rc = pcmk__ipc_msg_append(&cib_client->buffer, data);
 318 
 319     if (rc == pcmk_rc_ipc_more) {
 320         /* We haven't read the complete message yet, so just return. */
 321         return 0;
 322 
 323     } else if (rc == pcmk_rc_ok) {
 324         /* We've read the complete message and there's already a header on
 325          * the front.  Pass it off for processing.
 326          */
 327         op_request = pcmk__client_data2xml(cib_client, &id, &flags);
 328         g_byte_array_free(cib_client->buffer, TRUE);
 329         cib_client->buffer = NULL;
 330 
 331     } else {
 332         /* Some sort of error occurred reassembling the message.  All we can
 333          * do is clean up, log an error and return.
 334          */
 335         crm_err("Error when reading IPC message: %s", pcmk_rc_str(rc));
 336 
 337         if (cib_client->buffer != NULL) {
 338             g_byte_array_free(cib_client->buffer, TRUE);
 339             cib_client->buffer = NULL;
 340         }
 341 
 342         return 0;
 343     }
 344 
 345     if (op_request) {
 346         int rc = pcmk_rc_ok;
 347 
 348         rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 349                                 cib_none);
 350         if (rc != pcmk_rc_ok) {
 351             crm_warn("Couldn't parse options from request: %s",
 352                      pcmk_rc_str(rc));
 353         }
 354     }
 355 
 356     if (op_request == NULL) {
 357         crm_trace("Invalid message from %p", c);
 358         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL,
 359                            CRM_EX_PROTOCOL);
 360         return 0;
 361     }
 362 
 363     if (pcmk_is_set(call_options, cib_sync_call)) {
 364         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 365         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 366         cib_client->request_id = id;    /* Reply only to the last one */
 367     }
 368 
 369     if (cib_client->name == NULL) {
 370         const char *value = crm_element_value(op_request,
 371                                               PCMK__XA_CIB_CLIENTNAME);
 372 
 373         if (value == NULL) {
 374             cib_client->name = pcmk__itoa(cib_client->pid);
 375         } else {
 376             cib_client->name = pcmk__str_copy(value);
 377             if (pcmk__parse_server(value) != pcmk_ipc_unknown) {
 378                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 379             }
 380         }
 381     }
 382 
 383     /* Allow cluster daemons more leeway before being evicted */
 384     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 385         const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT);
 386 
 387         pcmk__set_client_queue_max(cib_client, qmax);
 388     }
 389 
 390     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTID, cib_client->id);
 391     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTNAME, cib_client->name);
 392 
 393     CRM_LOG_ASSERT(cib_client->user != NULL);
 394     pcmk__update_acl_user(op_request, PCMK__XA_CIB_USER, cib_client->user);
 395 
 396     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 397     pcmk__xml_free(op_request);
 398 
 399     return 0;
 400 }
 401 
 402 static uint64_t ping_seq = 0;
 403 static char *ping_digest = NULL;
 404 static bool ping_modified_since = FALSE;
 405 
 406 static gboolean
 407 cib_digester_cb(gpointer data)
     /*  */
 408 {
 409     if (based_is_primary) {
 410         char buffer[32];
 411         xmlNode *ping = pcmk__xe_create(NULL, PCMK__XE_PING);
 412 
 413         ping_seq++;
 414         free(ping_digest);
 415         ping_digest = NULL;
 416         ping_modified_since = FALSE;
 417         snprintf(buffer, 32, "%" PRIu64, ping_seq);
 418         crm_trace("Requesting peer digests (%s)", buffer);
 419 
 420         crm_xml_add(ping, PCMK__XA_T, PCMK__VALUE_CIB);
 421         crm_xml_add(ping, PCMK__XA_CIB_OP, CRM_OP_PING);
 422         crm_xml_add(ping, PCMK__XA_CIB_PING_ID, buffer);
 423 
 424         crm_xml_add(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
 425         pcmk__cluster_send_message(NULL, pcmk_ipc_based, ping);
 426 
 427         pcmk__xml_free(ping);
 428     }
 429     return FALSE;
 430 }
 431 
 432 static void
 433 process_ping_reply(xmlNode *reply) 
     /*  */
 434 {
 435     uint64_t seq = 0;
 436     const char *host = crm_element_value(reply, PCMK__XA_SRC);
 437 
 438     xmlNode *wrapper = pcmk__xe_first_child(reply, PCMK__XE_CIB_CALLDATA, NULL,
 439                                             NULL);
 440     xmlNode *pong = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 441 
 442     const char *seq_s = crm_element_value(pong, PCMK__XA_CIB_PING_ID);
 443     const char *digest = crm_element_value(pong, PCMK__XA_DIGEST);
 444 
 445     if (seq_s == NULL) {
 446         crm_debug("Ignoring ping reply with no " PCMK__XA_CIB_PING_ID);
 447         return;
 448 
 449     } else {
 450         long long seq_ll;
 451         int rc = pcmk__scan_ll(seq_s, &seq_ll, 0LL);
 452 
 453         if (rc != pcmk_rc_ok) {
 454             crm_debug("Ignoring ping reply with invalid " PCMK__XA_CIB_PING_ID
 455                       " '%s': %s", seq_s, pcmk_rc_str(rc));
 456             return;
 457         }
 458         seq = (uint64_t) seq_ll;
 459     }
 460 
 461     if(digest == NULL) {
 462         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 463 
 464     } else if(seq != ping_seq) {
 465         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 466 
 467     } else if(ping_modified_since) {
 468         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 469 
 470     } else {
 471         if(ping_digest == NULL) {
 472             crm_trace("Calculating new digest");
 473             ping_digest = pcmk__digest_xml(the_cib, true);
 474         }
 475 
 476         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 477         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 478             xmlNode *wrapper = pcmk__xe_first_child(pong, PCMK__XE_CIB_CALLDATA,
 479                                                     NULL, NULL);
 480             xmlNode *remote_cib = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 481 
 482             const char *admin_epoch_s = NULL;
 483             const char *epoch_s = NULL;
 484             const char *num_updates_s = NULL;
 485 
 486             if (remote_cib != NULL) {
 487                 admin_epoch_s = crm_element_value(remote_cib,
 488                                                   PCMK_XA_ADMIN_EPOCH);
 489                 epoch_s = crm_element_value(remote_cib, PCMK_XA_EPOCH);
 490                 num_updates_s = crm_element_value(remote_cib,
 491                                                   PCMK_XA_NUM_UPDATES);
 492             }
 493 
 494             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 495                        crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH),
 496                        crm_element_value(the_cib, PCMK_XA_EPOCH),
 497                        crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
 498                        ping_digest, host,
 499                        pcmk__s(admin_epoch_s, "_"),
 500                        pcmk__s(epoch_s, "_"),
 501                        pcmk__s(num_updates_s, "_"),
 502                        digest, remote_cib);
 503 
 504             if(remote_cib && remote_cib->children) {
 505                 // Additional debug
 506                 pcmk__xml_mark_changes(the_cib, remote_cib);
 507                 pcmk__log_xml_changes(LOG_INFO, remote_cib);
 508                 crm_trace("End of differences");
 509             }
 510 
 511             pcmk__xml_free(remote_cib);
 512             sync_our_cib(reply, FALSE);
 513         }
 514     }
 515 }
 516 
 517 static void
 518 parse_local_options(const pcmk__client_t *cib_client,
     /*  */
 519                     const cib__operation_t *operation,
 520                     const char *host, const char *op, gboolean *local_notify,
 521                     gboolean *needs_reply, gboolean *process,
 522                     gboolean *needs_forward)
 523 {
 524     // Process locally and notify local client
 525     *process = TRUE;
 526     *needs_reply = FALSE;
 527     *local_notify = TRUE;
 528     *needs_forward = FALSE;
 529 
 530     if (pcmk_is_set(operation->flags, cib__op_attr_local)) {
 531         /* Always process locally if cib__op_attr_local is set.
 532          *
 533          * @COMPAT: Currently host is ignored. At a compatibility break, throw
 534          * an error (from cib_process_request() or earlier) if host is not NULL or
 535          * OUR_NODENAME.
 536          */
 537         crm_trace("Processing always-local %s op from client %s",
 538                   op, pcmk__client_name(cib_client));
 539 
 540         if (!pcmk__str_eq(host, OUR_NODENAME,
 541                           pcmk__str_casei|pcmk__str_null_matches)) {
 542 
 543             crm_warn("Operation '%s' is always local but its target host is "
 544                      "set to '%s'",
 545                      op, host);
 546         }
 547         return;
 548     }
 549 
 550     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 551         || !pcmk__str_eq(host, OUR_NODENAME,
 552                          pcmk__str_casei|pcmk__str_null_matches)) {
 553 
 554         // Forward modifying and non-local requests via cluster
 555         *process = FALSE;
 556         *needs_reply = FALSE;
 557         *local_notify = FALSE;
 558         *needs_forward = TRUE;
 559 
 560         crm_trace("%s op from %s needs to be forwarded to %s",
 561                   op, pcmk__client_name(cib_client),
 562                   pcmk__s(host, "all nodes"));
 563         return;
 564     }
 565 
 566     if (stand_alone) {
 567         crm_trace("Processing %s op from client %s (stand-alone)",
 568                   op, pcmk__client_name(cib_client));
 569 
 570     } else {
 571         crm_trace("Processing %saddressed %s op from client %s",
 572                   ((host != NULL)? "locally " : "un"),
 573                   op, pcmk__client_name(cib_client));
 574     }
 575 }
 576 
 577 static gboolean
 578 parse_peer_options(const cib__operation_t *operation, xmlNode *request,
     /*  */
 579                    gboolean *local_notify, gboolean *needs_reply,
 580                    gboolean *process)
 581 {
 582     /* TODO: What happens when an update comes in after node A
 583      * requests the CIB from node B, but before it gets the reply (and
 584      * sends out the replace operation)?
 585      *
 586      * (This may no longer be relevant since legacy mode was dropped; need to
 587      * trace code more closely to check.)
 588      */
 589     const char *host = NULL;
 590     const char *delegated = crm_element_value(request,
 591                                               PCMK__XA_CIB_DELEGATED_FROM);
 592     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 593     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 594     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 595 
 596     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 597 
 598     if (originator == NULL) { // Shouldn't be possible
 599         originator = "peer";
 600     }
 601 
 602     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
 603         // sync_our_cib() sets PCMK__XA_CIB_ISREPLYTO
 604         if (reply_to) {
 605             delegated = reply_to;
 606         }
 607         goto skip_is_reply;
 608 
 609     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
 610                             pcmk__str_none)) {
 611         // Nothing to do
 612 
 613     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 614         process_ping_reply(request);
 615         return FALSE;
 616 
 617     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
 618         /* Only the DC (node with the oldest software) should process
 619          * this operation if PCMK__XA_CIB_SCHEMA_MAX is unset.
 620          *
 621          * If the DC is happy it will then send out another
 622          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
 623          * upgrade.
 624          *
 625          * Except this time PCMK__XA_CIB_SCHEMA_MAX will be set which puts a
 626          * limit on how far newer nodes will go
 627          */
 628         const char *max = crm_element_value(request, PCMK__XA_CIB_SCHEMA_MAX);
 629         const char *upgrade_rc = crm_element_value(request,
 630                                                    PCMK__XA_CIB_UPGRADE_RC);
 631 
 632         crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s",
 633                   (is_reply? "reply" : "request"),
 634                   (based_is_primary? "primary" : "secondary"),
 635                   pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none"));
 636 
 637         if (upgrade_rc != NULL) {
 638             // Our upgrade request was rejected by DC, notify clients of result
 639             crm_xml_add(request, PCMK__XA_CIB_RC, upgrade_rc);
 640 
 641         } else if ((max == NULL) && based_is_primary) {
 642             /* We are the DC, check if this upgrade is allowed */
 643             goto skip_is_reply;
 644 
 645         } else if(max) {
 646             /* Ok, go ahead and upgrade to 'max' */
 647             goto skip_is_reply;
 648 
 649         } else {
 650             // Ignore broadcast client requests when we're not primary
 651             return FALSE;
 652         }
 653 
 654     } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 655         crm_info("Detected legacy %s global update from %s", op, originator);
 656         send_sync_request(NULL);
 657         return FALSE;
 658 
 659     } else if (is_reply
 660                && pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
 661         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 662         return FALSE;
 663 
 664     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 665         *local_notify = FALSE;
 666         if (reply_to == NULL) {
 667             *process = TRUE;
 668         } else { // Not possible?
 669             crm_debug("Ignoring shutdown request from %s because reply_to=%s",
 670                       originator, reply_to);
 671         }
 672         return *process;
 673     }
 674 
 675     if (is_reply) {
 676         crm_trace("Will notify local clients for %s reply from %s",
 677                   op, originator);
 678         *process = FALSE;
 679         *needs_reply = FALSE;
 680         *local_notify = TRUE;
 681         return TRUE;
 682     }
 683 
 684   skip_is_reply:
 685     *process = TRUE;
 686     *needs_reply = FALSE;
 687 
 688     *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
 689 
 690     host = crm_element_value(request, PCMK__XA_CIB_HOST);
 691     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 692         crm_trace("Processing %s request sent to us from %s", op, originator);
 693         *needs_reply = TRUE;
 694         return TRUE;
 695 
 696     } else if (host != NULL) {
 697         crm_trace("Ignoring %s request intended for CIB manager on %s",
 698                   op, host);
 699         return FALSE;
 700 
 701     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 702         *needs_reply = TRUE;
 703     }
 704 
 705     crm_trace("Processing %s request broadcast by %s call %s on %s "
 706               "(local clients will%s be notified)", op,
 707               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CLIENTNAME),
 708                       "client"),
 709               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CALLID),
 710                       "without ID"),
 711               originator, (*local_notify? "" : "not"));
 712     return TRUE;
 713 }
 714 
 715 /*!
 716  * \internal
 717  * \brief Forward a CIB request to the appropriate target host(s)
 718  *
 719  * \param[in] request  CIB request to forward
 720  */
 721 static void
 722 forward_request(xmlNode *request)
     /*  */
 723 {
 724     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 725     const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
 726     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
 727     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 728     const char *client_name = crm_element_value(request,
 729                                                 PCMK__XA_CIB_CLIENTNAME);
 730     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
 731     pcmk__node_status_t *peer = NULL;
 732 
 733     int log_level = LOG_INFO;
 734 
 735     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
 736         log_level = LOG_DEBUG;
 737     }
 738 
 739     do_crm_log(log_level,
 740                "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
 741                pcmk__s(op, "invalid"),
 742                pcmk__s(section, "all"),
 743                pcmk__s(host, "all"),
 744                pcmk__s(originator, "local"),
 745                pcmk__s(client_name, "unspecified"),
 746                pcmk__s(call_id, "unspecified"));
 747 
 748     crm_xml_add(request, PCMK__XA_CIB_DELEGATED_FROM, OUR_NODENAME);
 749 
 750     if (host != NULL) {
 751         peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster_member);
 752     }
 753     pcmk__cluster_send_message(peer, pcmk_ipc_based, request);
 754 
 755     // Return the request to its original state
 756     pcmk__xe_remove_attr(request, PCMK__XA_CIB_DELEGATED_FROM);
 757 }
 758 
 759 static void
 760 send_peer_reply(xmlNode *msg, const char *originator)
     /*  */
 761 {
 762     const pcmk__node_status_t *node = NULL;
 763 
 764     if ((msg == NULL) || (originator == NULL)) {
 765         return;
 766     }
 767 
 768     // Send reply via cluster to originating node
 769     node = pcmk__get_node(0, originator, NULL,
 770                           pcmk__node_search_cluster_member);
 771 
 772     crm_trace("Sending request result to %s only", originator);
 773     crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator);
 774     pcmk__cluster_send_message(node, pcmk_ipc_based, msg);
 775 }
 776 
 777 /*!
 778  * \internal
 779  * \brief Handle an IPC or CPG message containing a request
 780  *
 781  * \param[in,out] request        Request XML
 782  * \param[in] privileged         Whether privileged commands may be run
 783  *                               (see cib_server_ops[] definition)
 784  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
 785  *
 786  * \return Legacy Pacemaker return code
 787  */
 788 int
 789 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
 790                     const pcmk__client_t *cib_client)
 791 {
 792     // @TODO: Break into multiple smaller functions
 793     uint32_t call_options = cib_none;
 794 
 795     gboolean process = TRUE;        // Whether to process request locally now
 796     gboolean is_update = TRUE;      // Whether request would modify CIB
 797     gboolean needs_reply = TRUE;    // Whether to build a reply
 798     gboolean local_notify = FALSE;  // Whether to notify (local) requester
 799     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 800 
 801     xmlNode *op_reply = NULL;
 802     xmlNode *result_diff = NULL;
 803 
 804     int rc = pcmk_ok;
 805     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 806     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 807     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
 808     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
 809     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
 810     const char *client_name = crm_element_value(request,
 811                                                 PCMK__XA_CIB_CLIENTNAME);
 812     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 813 
 814     const cib__operation_t *operation = NULL;
 815     cib__op_fn_t op_function = NULL;
 816 
 817     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
 818                             cib_none);
 819     if (rc != pcmk_rc_ok) {
 820         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
 821     }
 822 
 823     if ((host != NULL) && (*host == '\0')) {
 824         host = NULL;
 825     }
 826 
 827     if (cib_client == NULL) {
 828         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
 829                   op, pcmk__s(client_name, "client"), call_id, originator,
 830                   pcmk__s(host, "all"), reply_to);
 831     } else {
 832         crm_xml_add(request, PCMK__XA_SRC, OUR_NODENAME);
 833         crm_trace("Processing local %s operation from %s/%s intended for %s",
 834                   op, pcmk__s(client_name, "client"), call_id,
 835                   pcmk__s(host, "all"));
 836     }
 837 
 838     rc = cib__get_operation(op, &operation);
 839     rc = pcmk_rc2legacy(rc);
 840     if (rc != pcmk_ok) {
 841         /* TODO: construct error reply? */
 842         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
 843         return rc;
 844     }
 845 
 846     op_function = based_get_op_function(operation);
 847     if (op_function == NULL) {
 848         crm_err("Operation %s not supported by CIB manager", op);
 849         return -EOPNOTSUPP;
 850     }
 851 
 852     if (cib_client != NULL) {
 853         parse_local_options(cib_client, operation, host, op,
 854                             &local_notify, &needs_reply, &process,
 855                             &needs_forward);
 856 
 857     } else if (!parse_peer_options(operation, request, &local_notify,
 858                                    &needs_reply, &process)) {
 859         return rc;
 860     }
 861 
 862     if (pcmk_is_set(call_options, cib_transaction)) {
 863         /* All requests in a transaction are processed locally against a working
 864          * CIB copy, and we don't notify for individual requests because the
 865          * entire transaction is atomic.
 866          *
 867          * We still call the option parser functions above, for the sake of log
 868          * messages and checking whether we're the target for peer requests.
 869          */
 870         process = TRUE;
 871         needs_reply = FALSE;
 872         local_notify = FALSE;
 873         needs_forward = FALSE;
 874     }
 875 
 876     is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies);
 877 
 878     if (pcmk_is_set(call_options, cib_discard_reply)) {
 879         /* If the request will modify the CIB, and we are in legacy mode, we
 880          * need to build a reply so we can broadcast a diff, even if the
 881          * requester doesn't want one.
 882          */
 883         needs_reply = FALSE;
 884         local_notify = FALSE;
 885         crm_trace("Client is not interested in the reply");
 886     }
 887 
 888     if (needs_forward) {
 889         forward_request(request);
 890         return rc;
 891     }
 892 
 893     if (cib_status != pcmk_ok) {
 894         rc = cib_status;
 895         crm_err("Ignoring request because cluster configuration is invalid "
 896                 "(please repair and restart): %s", pcmk_strerror(rc));
 897         op_reply = create_cib_reply(op, call_id, client_id, call_options, rc,
 898                                     the_cib);
 899 
 900     } else if (process) {
 901         time_t finished = 0;
 902         time_t now = time(NULL);
 903         int level = LOG_INFO;
 904         const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
 905         const char *admin_epoch_s = NULL;
 906         const char *epoch_s = NULL;
 907         const char *num_updates_s = NULL;
 908 
 909         rc = cib_process_command(request, operation, op_function, &op_reply,
 910                                  &result_diff, privileged);
 911 
 912         if (!is_update) {
 913             level = LOG_TRACE;
 914 
 915         } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 916             switch (rc) {
 917                 case pcmk_ok:
 918                     level = LOG_INFO;
 919                     break;
 920                 case -pcmk_err_old_data:
 921                 case -pcmk_err_diff_resync:
 922                 case -pcmk_err_diff_failed:
 923                     level = LOG_TRACE;
 924                     break;
 925                 default:
 926                     level = LOG_ERR;
 927             }
 928 
 929         } else if (rc != pcmk_ok) {
 930             level = LOG_WARNING;
 931         }
 932 
 933         if (the_cib != NULL) {
 934             admin_epoch_s = crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH);
 935             epoch_s = crm_element_value(the_cib, PCMK_XA_EPOCH);
 936             num_updates_s = crm_element_value(the_cib, PCMK_XA_NUM_UPDATES);
 937         }
 938 
 939         do_crm_log(level,
 940                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
 941                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
 942                    originator ? originator : "local",
 943                    pcmk__s(client_name, "client"), call_id,
 944                    pcmk__s(admin_epoch_s, "0"),
 945                    pcmk__s(epoch_s, "0"),
 946                    pcmk__s(num_updates_s, "0"));
 947 
 948         finished = time(NULL);
 949         if ((finished - now) > 3) {
 950             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
 951             crm_write_blackbox(0, NULL);
 952         }
 953 
 954         if (op_reply == NULL && (needs_reply || local_notify)) {
 955             crm_err("Unexpected NULL reply to message");
 956             crm_log_xml_err(request, "null reply");
 957             needs_reply = FALSE;
 958             local_notify = FALSE;
 959         }
 960     }
 961 
 962     if (is_update) {
 963         crm_trace("Completed pre-sync update from %s/%s/%s%s",
 964                   originator ? originator : "local",
 965                   pcmk__s(client_name, "client"), call_id,
 966                   local_notify?" with local notification":"");
 967 
 968     } else if (!needs_reply || stand_alone) {
 969         // This was a non-originating secondary update
 970         crm_trace("Completed update as secondary");
 971 
 972     } else if ((cib_client == NULL)
 973                && !pcmk_is_set(call_options, cib_discard_reply)) {
 974 
 975         if (is_update == FALSE || result_diff == NULL) {
 976             crm_trace("Request not broadcast: R/O call");
 977 
 978         } else if (rc != pcmk_ok) {
 979             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
 980 
 981         } else {
 982             crm_trace("Directing reply to %s", originator);
 983         }
 984 
 985         send_peer_reply(op_reply, originator);
 986     }
 987 
 988     if (local_notify && client_id) {
 989         crm_trace("Performing local %ssync notification for %s",
 990                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
 991                   client_id);
 992         if (process == FALSE) {
 993             do_local_notify(request, client_id,
 994                             pcmk_is_set(call_options, cib_sync_call),
 995                             (cib_client == NULL));
 996         } else {
 997             do_local_notify(op_reply, client_id,
 998                             pcmk_is_set(call_options, cib_sync_call),
 999                             (cib_client == NULL));
1000         }
1001     }
1002 
1003     pcmk__xml_free(op_reply);
1004     pcmk__xml_free(result_diff);
1005 
1006     return rc;
1007 }
1008 
1009 /*!
1010  * \internal
1011  * \brief Get a CIB operation's input from the request XML
1012  *
1013  * \param[in]  request  CIB request XML
1014  * \param[in]  type     CIB operation type
1015  * \param[out] section  Where to store CIB section name
1016  *
1017  * \return Input XML for CIB operation
1018  *
1019  * \note If not \c NULL, the return value is a non-const pointer to part of
1020  *       \p request. The caller should not free it directly.
1021  */
1022 static xmlNode *
1023 prepare_input(const xmlNode *request, enum cib__op_type type,
     /*  */
1024               const char **section)
1025 {
1026     xmlNode *wrapper = pcmk__xe_first_child(request, PCMK__XE_CIB_CALLDATA,
1027                                             NULL, NULL);
1028     xmlNode *input = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
1029 
1030     if (type == cib__op_apply_patch) {
1031         *section = NULL;
1032     } else {
1033         *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
1034     }
1035 
1036     // Grab the specified section
1037     if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) {
1038         input = pcmk_find_cib_element(input, *section);
1039     }
1040 
1041     return input;
1042 }
1043 
1044 #define XPATH_CONFIG_CHANGE         \
1045     "//" PCMK_XE_CHANGE             \
1046     "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]"
1047 
1048 static bool
1049 contains_config_change(xmlNode *diff)
     /*  */
1050 {
1051     bool changed = false;
1052 
1053     if (diff) {
1054         xmlXPathObject *xpathObj = pcmk__xpath_search(diff->doc,
1055                                                       XPATH_CONFIG_CHANGE);
1056 
1057         if (pcmk__xpath_num_results(xpathObj) > 0) {
1058             changed = true;
1059         }
1060         xmlXPathFreeObject(xpathObj);
1061     }
1062     return changed;
1063 }
1064 
1065 static int
1066 cib_process_command(xmlNode *request, const cib__operation_t *operation,
     /*  */
1067                     cib__op_fn_t op_function, xmlNode **reply,
1068                     xmlNode **cib_diff, bool privileged)
1069 {
1070     xmlNode *input = NULL;
1071     xmlNode *output = NULL;
1072     xmlNode *result_cib = NULL;
1073 
1074     uint32_t call_options = cib_none;
1075 
1076     const char *op = NULL;
1077     const char *section = NULL;
1078     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
1079     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
1080     const char *client_name = crm_element_value(request,
1081                                                 PCMK__XA_CIB_CLIENTNAME);
1082     const char *originator = crm_element_value(request, PCMK__XA_SRC);
1083 
1084     int rc = pcmk_ok;
1085 
1086     bool config_changed = false;
1087     bool manage_counters = true;
1088 
1089     static mainloop_timer_t *digest_timer = NULL;
1090 
1091     pcmk__assert(cib_status == pcmk_ok);
1092 
1093     if(digest_timer == NULL) {
1094         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1095     }
1096 
1097     *reply = NULL;
1098     *cib_diff = NULL;
1099 
1100     /* Start processing the request... */
1101     op = crm_element_value(request, PCMK__XA_CIB_OP);
1102     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
1103                             cib_none);
1104     if (rc != pcmk_rc_ok) {
1105         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
1106     }
1107 
1108     if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) {
1109         rc = -EACCES;
1110         crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc));
1111         goto done;
1112     }
1113 
1114     input = prepare_input(request, operation->type, &section);
1115 
1116     if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
1117         rc = cib_perform_op(NULL, op, call_options, op_function, true, section,
1118                             request, input, false, &config_changed, &the_cib,
1119                             &result_cib, NULL, &output);
1120 
1121         CRM_CHECK(result_cib == NULL, pcmk__xml_free(result_cib));
1122         goto done;
1123     }
1124 
1125     /* @COMPAT: Handle a valid write action (legacy)
1126      *
1127      * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff
1128      * portion is. However, PCMK__XA_CIB_UPDATE may be set by a sync operation
1129      * even in non-legacy mode, and manage_counters tells xml_create_patchset()
1130      * whether to update version/epoch info.
1131      */
1132     if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
1133         manage_counters = false;
1134         cib__set_call_options(call_options, "call", cib_force_diff);
1135         crm_trace("Global update detected");
1136 
1137         CRM_LOG_ASSERT(pcmk__str_any_of(op,
1138                                         PCMK__CIB_REQUEST_APPLY_PATCH,
1139                                         PCMK__CIB_REQUEST_REPLACE,
1140                                         NULL));
1141     }
1142 
1143     ping_modified_since = TRUE;
1144 
1145     // result_cib must not be modified after cib_perform_op() returns
1146     rc = cib_perform_op(NULL, op, call_options, op_function, false, section,
1147                         request, input, manage_counters, &config_changed,
1148                         &the_cib, &result_cib, cib_diff, &output);
1149 
1150     /* Always write to disk for successful ops with the flag set. This also
1151      * negates the need to detect ordering changes.
1152      */
1153     if ((rc == pcmk_ok)
1154         && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1155 
1156         config_changed = true;
1157     }
1158 
1159     if ((rc == pcmk_ok)
1160         && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) {
1161 
1162         if (result_cib != the_cib) {
1163             if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1164                 config_changed = true;
1165             }
1166 
1167             crm_trace("Activating %s->%s%s",
1168                       crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
1169                       crm_element_value(result_cib, PCMK_XA_NUM_UPDATES),
1170                       (config_changed? " changed" : ""));
1171 
1172             rc = activateCibXml(result_cib, config_changed, op);
1173             if (rc != pcmk_ok) {
1174                 crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc));
1175             }
1176         }
1177 
1178         if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
1179             cib_read_config(config_hash, result_cib);
1180         }
1181 
1182         /* @COMPAT Nodes older than feature set 3.19.0 don't support
1183          * transactions. In a mixed-version cluster with nodes <3.19.0, we must
1184          * sync the updated CIB, so that the older nodes receive the changes.
1185          * Any node that has already applied the transaction will ignore the
1186          * synced CIB.
1187          *
1188          * To ensure the updated CIB is synced from only one node, we sync it
1189          * from the originator.
1190          */
1191         if ((operation->type == cib__op_commit_transact)
1192             && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei)
1193             && compare_version(crm_element_value(the_cib,
1194                                                  PCMK_XA_CRM_FEATURE_SET),
1195                                "3.19.0") < 0) {
1196 
1197             sync_our_cib(request, TRUE);
1198         }
1199 
1200         mainloop_timer_stop(digest_timer);
1201         mainloop_timer_start(digest_timer);
1202 
1203     } else if (rc == -pcmk_err_schema_validation) {
1204         pcmk__assert(result_cib != the_cib);
1205 
1206         if (output != NULL) {
1207             crm_log_xml_info(output, "cib:output");
1208             pcmk__xml_free(output);
1209         }
1210 
1211         output = result_cib;
1212 
1213     } else {
1214         crm_trace("Not activating %d %d %s", rc,
1215                   pcmk_is_set(call_options, cib_dryrun),
1216                   crm_element_value(result_cib, PCMK_XA_NUM_UPDATES));
1217 
1218         if (result_cib != the_cib) {
1219             pcmk__xml_free(result_cib);
1220         }
1221     }
1222 
1223     if (!pcmk_any_flags_set(call_options,
1224                             cib_dryrun|cib_inhibit_notify|cib_transaction)) {
1225         crm_trace("Sending notifications %d",
1226                   pcmk_is_set(call_options, cib_dryrun));
1227         cib_diff_notify(op, rc, call_id, client_id, client_name, originator,
1228                         input, *cib_diff);
1229     }
1230 
1231     pcmk__log_xml_patchset(LOG_TRACE, *cib_diff);
1232 
1233   done:
1234     if (!pcmk_is_set(call_options, cib_discard_reply)) {
1235         *reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1236                                   output);
1237     }
1238 
1239     if (output != the_cib) {
1240         pcmk__xml_free(output);
1241     }
1242     crm_trace("done");
1243     return rc;
1244 }
1245 
1246 void
1247 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1248 {
1249     const char *reason = NULL;
1250     const char *originator = crm_element_value(msg, PCMK__XA_SRC);
1251 
1252     if (pcmk__peer_cache == NULL) {
1253         reason = "membership not established";
1254         goto bail;
1255     }
1256 
1257     if (crm_element_value(msg, PCMK__XA_CIB_CLIENTNAME) == NULL) {
1258         crm_xml_add(msg, PCMK__XA_CIB_CLIENTNAME, originator);
1259     }
1260 
1261     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1262     cib_process_request(msg, TRUE, NULL);
1263     return;
1264 
1265   bail:
1266     if (reason) {
1267         const char *op = crm_element_value(msg, PCMK__XA_CIB_OP);
1268 
1269         crm_warn("Discarding %s message from %s: %s", op, originator, reason);
1270     }
1271 }
1272 
1273 static gboolean
1274 cib_force_exit(gpointer data)
     /*  */
1275 {
1276     crm_notice("Exiting immediately after %s without shutdown acknowledgment",
1277                pcmk__readable_interval(EXIT_ESCALATION_MS));
1278     terminate_cib(CRM_EX_ERROR);
1279     return FALSE;
1280 }
1281 
1282 static void
1283 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1284 {
1285     pcmk__client_t *a_client = value;
1286 
1287     crm_err("Can't disconnect client %s: Not implemented",
1288             pcmk__client_name(a_client));
1289 }
1290 
1291 static void
1292 initiate_exit(void)
     /*  */
1293 {
1294     int active = 0;
1295     xmlNode *leaving = NULL;
1296 
1297     active = pcmk__cluster_num_active_nodes();
1298     if (active < 2) { // This is the last active node
1299         crm_info("Exiting without sending shutdown request (no active peers)");
1300         terminate_cib(CRM_EX_OK);
1301         return;
1302     }
1303 
1304     crm_info("Sending shutdown request to %d peers", active);
1305 
1306     leaving = pcmk__xe_create(NULL, PCMK__XE_EXIT_NOTIFICATION);
1307     crm_xml_add(leaving, PCMK__XA_T, PCMK__VALUE_CIB);
1308     crm_xml_add(leaving, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_SHUTDOWN);
1309 
1310     pcmk__cluster_send_message(NULL, pcmk_ipc_based, leaving);
1311     pcmk__xml_free(leaving);
1312 
1313     pcmk__create_timer(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1314 }
1315 
1316 void
1317 cib_shutdown(int nsig)
     /*  */
1318 {
1319     struct qb_ipcs_stats srv_stats;
1320 
1321     if (cib_shutdown_flag == FALSE) {
1322         int disconnects = 0;
1323         qb_ipcs_connection_t *c = NULL;
1324 
1325         cib_shutdown_flag = TRUE;
1326 
1327         c = qb_ipcs_connection_first_get(ipcs_rw);
1328         while (c != NULL) {
1329             qb_ipcs_connection_t *last = c;
1330 
1331             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1332 
1333             crm_debug("Disconnecting r/w client %p...", last);
1334             qb_ipcs_disconnect(last);
1335             qb_ipcs_connection_unref(last);
1336             disconnects++;
1337         }
1338 
1339         c = qb_ipcs_connection_first_get(ipcs_ro);
1340         while (c != NULL) {
1341             qb_ipcs_connection_t *last = c;
1342 
1343             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1344 
1345             crm_debug("Disconnecting r/o client %p...", last);
1346             qb_ipcs_disconnect(last);
1347             qb_ipcs_connection_unref(last);
1348             disconnects++;
1349         }
1350 
1351         c = qb_ipcs_connection_first_get(ipcs_shm);
1352         while (c != NULL) {
1353             qb_ipcs_connection_t *last = c;
1354 
1355             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1356 
1357             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1358             qb_ipcs_disconnect(last);
1359             qb_ipcs_connection_unref(last);
1360             disconnects++;
1361         }
1362 
1363         disconnects += pcmk__ipc_client_count();
1364 
1365         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1366         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1367         crm_info("Disconnected %d clients", disconnects);
1368     }
1369 
1370     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1371 
1372     if (pcmk__ipc_client_count() == 0) {
1373         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1374         initiate_exit();
1375 
1376     } else {
1377         crm_info("Waiting on %d clients to disconnect (%d)",
1378                  pcmk__ipc_client_count(), srv_stats.active_connections);
1379     }
1380 }
1381 
1382 extern int remote_fd;
1383 extern int remote_tls_fd;
1384 
1385 /*!
1386  * \internal
1387  * \brief Close remote sockets, free the global CIB and quit
1388  *
1389  * \param[in] exit_status  What exit status to use (if -1, use CRM_EX_OK, but
1390  *                         skip disconnecting from the cluster layer)
1391  */
1392 void
1393 terminate_cib(int exit_status)
     /*  */
1394 {
1395     if (remote_fd > 0) {
1396         close(remote_fd);
1397         remote_fd = 0;
1398     }
1399     if (remote_tls_fd > 0) {
1400         close(remote_tls_fd);
1401         remote_tls_fd = 0;
1402     }
1403 
1404     uninitializeCib();
1405 
1406     // Exit immediately on error
1407     if (exit_status > CRM_EX_OK) {
1408         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1409         crm_exit(exit_status);
1410         return;
1411     }
1412 
1413     if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1414         /* Quit via returning from the main loop. If exit_status has the special
1415          * value -1, we skip the disconnect here, and it will be done when the
1416          * main loop returns (this allows the peer status callback to avoid
1417          * messing with the peer caches).
1418          */
1419         if (exit_status == CRM_EX_OK) {
1420             pcmk_cluster_disconnect(crm_cluster);
1421         }
1422         g_main_loop_quit(mainloop);
1423         return;
1424     }
1425 
1426     /* Exit cleanly. Even the peer status callback can disconnect here, because
1427      * we're not returning control to the caller.
1428      */
1429     pcmk_cluster_disconnect(crm_cluster);
1430     pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1431     crm_exit(CRM_EX_OK);
1432 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
