daemons/based/based

/* */
This source file includes following definitions.
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
create_cib_reply
do_local_notify
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
parse_local_options
parse_peer_options
forward_request
send_peer_reply
cib_process_request
prepare_input
contains_config_change
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
initiate_exit
cib_shutdown
terminate_cib
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>   // PRIu64
  22 
  23 #include <glib.h>
  24 #include <libxml/tree.h>
  25 
  26 #include <crm/crm.h>
  27 #include <crm/cib.h>
  28 #include <crm/cluster/internal.h>
  29 
  30 #include <crm/common/xml.h>
  31 #include <crm/common/remote_internal.h>
  32 
  33 #include <pacemaker-based.h>
  34 
  35 #define EXIT_ESCALATION_MS 10000
  36 
  37 qb_ipcs_service_t *ipcs_ro = NULL;
  38 qb_ipcs_service_t *ipcs_rw = NULL;
  39 qb_ipcs_service_t *ipcs_shm = NULL;
  40 
  41 static int cib_process_command(xmlNode *request,
  42                                const cib__operation_t *operation,
  43                                cib__op_fn_t op_function, xmlNode **reply,
  44                                xmlNode **cib_diff, bool privileged);
  45 
  46 static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
  47                                     size_t size, gboolean privileged);
  48 
  49 static int32_t
  50 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  51 {
  52     if (cib_shutdown_flag) {
  53         crm_info("Ignoring new IPC client [%d] during shutdown",
  54                  pcmk__client_pid(c));
  55         return -ECONNREFUSED;
  56     }
  57 
  58     if (pcmk__new_client(c, uid, gid) == NULL) {
  59         return -ENOMEM;
  60     }
  61     return 0;
  62 }
  63 
  64 static int32_t
  65 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  66 {
  67     pcmk__client_t *client = pcmk__find_client(c);
  68 
  69     crm_trace("%p message from %s", c, client->id);
  70     return cib_common_callback(c, data, size, TRUE);
  71 }
  72 
  73 static int32_t
  74 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  75 {
  76     pcmk__client_t *client = pcmk__find_client(c);
  77 
  78     crm_trace("%p message from %s", c, client->id);
  79     return cib_common_callback(c, data, size, FALSE);
  80 }
  81 
  82 /* Error code means? */
  83 static int32_t
  84 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
  85 {
  86     pcmk__client_t *client = pcmk__find_client(c);
  87 
  88     if (client == NULL) {
  89         return 0;
  90     }
  91     crm_trace("Connection %p", c);
  92     pcmk__free_client(client);
  93     return 0;
  94 }
  95 
  96 static void
  97 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
  98 {
  99     crm_trace("Connection %p", c);
 100     cib_ipc_closed(c);
 101     if (cib_shutdown_flag) {
 102         cib_shutdown(0);
 103     }
 104 }
 105 
 106 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 107     .connection_accept = cib_ipc_accept,
 108     .connection_created = NULL,
 109     .msg_process = cib_ipc_dispatch_ro,
 110     .connection_closed = cib_ipc_closed,
 111     .connection_destroyed = cib_ipc_destroy
 112 };
 113 
 114 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 115     .connection_accept = cib_ipc_accept,
 116     .connection_created = NULL,
 117     .msg_process = cib_ipc_dispatch_rw,
 118     .connection_closed = cib_ipc_closed,
 119     .connection_destroyed = cib_ipc_destroy
 120 };
 121 
 122 /*!
 123  * \internal
 124  * \brief Create reply XML for a CIB request
 125  *
 126  * \param[in] op            CIB operation type
 127  * \param[in] call_id       CIB call ID
 128  * \param[in] client_id     CIB client ID
 129  * \param[in] call_options  Group of <tt>enum cib_call_options</tt> flags
 130  * \param[in] rc            Request return code
 131  * \param[in] call_data     Request output data
 132  *
 133  * \return Reply XML (guaranteed not to be \c NULL)
 134  *
 135  * \note The caller is responsible for freeing the return value using
 136  *       \p pcmk__xml_free().
 137  */
 138 static xmlNode *
 139 create_cib_reply(const char *op, const char *call_id, const char *client_id,
     /*  */
 140                  uint32_t call_options, int rc, xmlNode *call_data)
 141 {
 142     xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_CIB_REPLY);
 143 
 144     crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_CIB);
 145     crm_xml_add(reply, PCMK__XA_CIB_OP, op);
 146     crm_xml_add(reply, PCMK__XA_CIB_CALLID, call_id);
 147     crm_xml_add(reply, PCMK__XA_CIB_CLIENTID, client_id);
 148     crm_xml_add_int(reply, PCMK__XA_CIB_CALLOPT, call_options);
 149     crm_xml_add_int(reply, PCMK__XA_CIB_RC, rc);
 150 
 151     if (call_data != NULL) {
 152         xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_CIB_CALLDATA);
 153 
 154         crm_trace("Attaching reply output");
 155         pcmk__xml_copy(wrapper, call_data);
 156     }
 157 
 158     crm_log_xml_explicit(reply, "cib:reply");
 159     return reply;
 160 }
 161 
 162 static void
 163 do_local_notify(const xmlNode *notify_src, const char *client_id,
     /*  */
 164                 bool sync_reply, bool from_peer)
 165 {
 166     int msg_id = 0;
 167     int rc = pcmk_rc_ok;
 168     pcmk__client_t *client_obj = NULL;
 169     uint32_t flags = crm_ipc_server_event;
 170 
 171     CRM_CHECK((notify_src != NULL) && (client_id != NULL), return);
 172 
 173     crm_element_value_int(notify_src, PCMK__XA_CIB_CALLID, &msg_id);
 174 
 175     client_obj = pcmk__find_client_by_id(client_id);
 176     if (client_obj == NULL) {
 177         crm_debug("Could not notify client %s%s %s of call %d result: "
 178                   "client no longer exists", client_id,
 179                   (from_peer? " (originator of delegated request)" : ""),
 180                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 181         return;
 182     }
 183 
 184     if (sync_reply) {
 185         flags = crm_ipc_flags_none;
 186         if (client_obj->ipcs != NULL) {
 187             msg_id = client_obj->request_id;
 188             client_obj->request_id = 0;
 189         }
 190     }
 191 
 192     switch (PCMK__CLIENT_TYPE(client_obj)) {
 193         case pcmk__client_ipc:
 194             rc = pcmk__ipc_send_xml(client_obj, msg_id, notify_src, flags);
 195             break;
 196         case pcmk__client_tls:
 197         case pcmk__client_tcp:
 198             rc = pcmk__remote_send_xml(client_obj->remote, notify_src);
 199             break;
 200         default:
 201             rc = EPROTONOSUPPORT;
 202             break;
 203     }
 204     if (rc == pcmk_rc_ok) {
 205         crm_trace("Notified %s client %s%s %s of call %d result",
 206                   pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 207                   pcmk__client_name(client_obj),
 208                   (from_peer? " (originator of delegated request)" : ""),
 209                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 210     } else {
 211         crm_warn("Could not notify %s client %s%s %s of call %d result: %s",
 212                  pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 213                  pcmk__client_name(client_obj),
 214                  (from_peer? " (originator of delegated request)" : ""),
 215                  (sync_reply? "synchronously" : "asynchronously"), msg_id,
 216                  pcmk_rc_str(rc));
 217     }
 218 }
 219 
 220 void
 221 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 222                            pcmk__client_t *cib_client, gboolean privileged)
 223 {
 224     const char *op = crm_element_value(op_request, PCMK__XA_CIB_OP);
 225     uint32_t call_options = cib_none;
 226     int rc = pcmk_rc_ok;
 227 
 228     rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 229                             cib_none);
 230     if (rc != pcmk_rc_ok) {
 231         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
 232     }
 233 
 234     /* Requests with cib_transaction set should not be sent to based directly
 235      * (outside of a commit-transaction request)
 236      */
 237     if (pcmk_is_set(call_options, cib_transaction)) {
 238         return;
 239     }
 240 
 241     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 242         if (flags & crm_ipc_client_response) {
 243             xmlNode *ack = pcmk__xe_create(NULL, __func__);
 244 
 245             crm_xml_add(ack, PCMK__XA_CIB_OP, CRM_OP_REGISTER);
 246             crm_xml_add(ack, PCMK__XA_CIB_CLIENTID, cib_client->id);
 247             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 248             cib_client->request_id = 0;
 249             pcmk__xml_free(ack);
 250         }
 251         return;
 252 
 253     } else if (pcmk__str_eq(op, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) {
 254         /* Update the notify filters for this client */
 255         int on_off = 0;
 256         crm_exit_t status = CRM_EX_OK;
 257         uint64_t bit = UINT64_C(0);
 258         const char *type = crm_element_value(op_request,
 259                                              PCMK__XA_CIB_NOTIFY_TYPE);
 260 
 261         crm_element_value_int(op_request, PCMK__XA_CIB_NOTIFY_ACTIVATE,
 262                               &on_off);
 263 
 264         crm_debug("Setting %s callbacks %s for client %s",
 265                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 266 
 267         if (pcmk__str_eq(type, PCMK__VALUE_CIB_POST_NOTIFY, pcmk__str_none)) {
 268             bit = cib_notify_post;
 269 
 270         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_PRE_NOTIFY,
 271                                 pcmk__str_none)) {
 272             bit = cib_notify_pre;
 273 
 274         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_UPDATE_CONFIRMATION,
 275                                 pcmk__str_none)) {
 276             bit = cib_notify_confirm;
 277 
 278         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_DIFF_NOTIFY,
 279                                 pcmk__str_none)) {
 280             bit = cib_notify_diff;
 281 
 282         } else {
 283             status = CRM_EX_INVALID_PARAM;
 284         }
 285 
 286         if (bit != 0) {
 287             if (on_off) {
 288                 pcmk__set_client_flags(cib_client, bit);
 289             } else {
 290                 pcmk__clear_client_flags(cib_client, bit);
 291             }
 292         }
 293 
 294         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status);
 295         return;
 296     }
 297 
 298     cib_process_request(op_request, privileged, cib_client);
 299 }
 300 
 301 int32_t
 302 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 303 {
 304     uint32_t id = 0;
 305     uint32_t flags = 0;
 306     uint32_t call_options = cib_none;
 307     pcmk__client_t *cib_client = pcmk__find_client(c);
 308     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 309 
 310     if (op_request) {
 311         int rc = pcmk_rc_ok;
 312 
 313         rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 314                                 cib_none);
 315         if (rc != pcmk_rc_ok) {
 316             crm_warn("Couldn't parse options from request: %s",
 317                      pcmk_rc_str(rc));
 318         }
 319     }
 320 
 321     if (op_request == NULL) {
 322         crm_trace("Invalid message from %p", c);
 323         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL,
 324                            CRM_EX_PROTOCOL);
 325         return 0;
 326 
 327     } else if(cib_client == NULL) {
 328         crm_trace("Invalid client %p", c);
 329         return 0;
 330     }
 331 
 332     if (pcmk_is_set(call_options, cib_sync_call)) {
 333         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 334         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 335         cib_client->request_id = id;    /* Reply only to the last one */
 336     }
 337 
 338     if (cib_client->name == NULL) {
 339         const char *value = crm_element_value(op_request,
 340                                               PCMK__XA_CIB_CLIENTNAME);
 341 
 342         if (value == NULL) {
 343             cib_client->name = pcmk__itoa(cib_client->pid);
 344         } else {
 345             cib_client->name = pcmk__str_copy(value);
 346             if (pcmk__parse_server(value) != pcmk_ipc_unknown) {
 347                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 348             }
 349         }
 350     }
 351 
 352     /* Allow cluster daemons more leeway before being evicted */
 353     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 354         const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT);
 355 
 356         pcmk__set_client_queue_max(cib_client, qmax);
 357     }
 358 
 359     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTID, cib_client->id);
 360     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTNAME, cib_client->name);
 361 
 362     CRM_LOG_ASSERT(cib_client->user != NULL);
 363     pcmk__update_acl_user(op_request, PCMK__XA_CIB_USER, cib_client->user);
 364 
 365     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 366     pcmk__xml_free(op_request);
 367 
 368     return 0;
 369 }
 370 
 371 static uint64_t ping_seq = 0;
 372 static char *ping_digest = NULL;
 373 static bool ping_modified_since = FALSE;
 374 
 375 static gboolean
 376 cib_digester_cb(gpointer data)
     /*  */
 377 {
 378     if (based_is_primary) {
 379         char buffer[32];
 380         xmlNode *ping = pcmk__xe_create(NULL, PCMK__XE_PING);
 381 
 382         ping_seq++;
 383         free(ping_digest);
 384         ping_digest = NULL;
 385         ping_modified_since = FALSE;
 386         snprintf(buffer, 32, "%" PRIu64, ping_seq);
 387         crm_trace("Requesting peer digests (%s)", buffer);
 388 
 389         crm_xml_add(ping, PCMK__XA_T, PCMK__VALUE_CIB);
 390         crm_xml_add(ping, PCMK__XA_CIB_OP, CRM_OP_PING);
 391         crm_xml_add(ping, PCMK__XA_CIB_PING_ID, buffer);
 392 
 393         crm_xml_add(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
 394         pcmk__cluster_send_message(NULL, pcmk_ipc_based, ping);
 395 
 396         pcmk__xml_free(ping);
 397     }
 398     return FALSE;
 399 }
 400 
 401 static void
 402 process_ping_reply(xmlNode *reply) 
     /*  */
 403 {
 404     uint64_t seq = 0;
 405     const char *host = crm_element_value(reply, PCMK__XA_SRC);
 406 
 407     xmlNode *wrapper = pcmk__xe_first_child(reply, PCMK__XE_CIB_CALLDATA, NULL,
 408                                             NULL);
 409     xmlNode *pong = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 410 
 411     const char *seq_s = crm_element_value(pong, PCMK__XA_CIB_PING_ID);
 412     const char *digest = crm_element_value(pong, PCMK__XA_DIGEST);
 413 
 414     if (seq_s == NULL) {
 415         crm_debug("Ignoring ping reply with no " PCMK__XA_CIB_PING_ID);
 416         return;
 417 
 418     } else {
 419         long long seq_ll;
 420         int rc = pcmk__scan_ll(seq_s, &seq_ll, 0LL);
 421 
 422         if (rc != pcmk_rc_ok) {
 423             crm_debug("Ignoring ping reply with invalid " PCMK__XA_CIB_PING_ID
 424                       " '%s': %s", seq_s, pcmk_rc_str(rc));
 425             return;
 426         }
 427         seq = (uint64_t) seq_ll;
 428     }
 429 
 430     if(digest == NULL) {
 431         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 432 
 433     } else if(seq != ping_seq) {
 434         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 435 
 436     } else if(ping_modified_since) {
 437         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 438 
 439     } else {
 440         if(ping_digest == NULL) {
 441             crm_trace("Calculating new digest");
 442             ping_digest = pcmk__digest_xml(the_cib, true);
 443         }
 444 
 445         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 446         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 447             xmlNode *wrapper = pcmk__xe_first_child(pong, PCMK__XE_CIB_CALLDATA,
 448                                                     NULL, NULL);
 449             xmlNode *remote_cib = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 450 
 451             const char *admin_epoch_s = NULL;
 452             const char *epoch_s = NULL;
 453             const char *num_updates_s = NULL;
 454 
 455             if (remote_cib != NULL) {
 456                 admin_epoch_s = crm_element_value(remote_cib,
 457                                                   PCMK_XA_ADMIN_EPOCH);
 458                 epoch_s = crm_element_value(remote_cib, PCMK_XA_EPOCH);
 459                 num_updates_s = crm_element_value(remote_cib,
 460                                                   PCMK_XA_NUM_UPDATES);
 461             }
 462 
 463             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 464                        crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH),
 465                        crm_element_value(the_cib, PCMK_XA_EPOCH),
 466                        crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
 467                        ping_digest, host,
 468                        pcmk__s(admin_epoch_s, "_"),
 469                        pcmk__s(epoch_s, "_"),
 470                        pcmk__s(num_updates_s, "_"),
 471                        digest, remote_cib);
 472 
 473             if(remote_cib && remote_cib->children) {
 474                 // Additional debug
 475                 xml_calculate_changes(the_cib, remote_cib);
 476                 pcmk__log_xml_changes(LOG_INFO, remote_cib);
 477                 crm_trace("End of differences");
 478             }
 479 
 480             pcmk__xml_free(remote_cib);
 481             sync_our_cib(reply, FALSE);
 482         }
 483     }
 484 }
 485 
 486 static void
 487 parse_local_options(const pcmk__client_t *cib_client,
     /*  */
 488                     const cib__operation_t *operation,
 489                     const char *host, const char *op, gboolean *local_notify,
 490                     gboolean *needs_reply, gboolean *process,
 491                     gboolean *needs_forward)
 492 {
 493     // Process locally and notify local client
 494     *process = TRUE;
 495     *needs_reply = FALSE;
 496     *local_notify = TRUE;
 497     *needs_forward = FALSE;
 498 
 499     if (pcmk_is_set(operation->flags, cib__op_attr_local)) {
 500         /* Always process locally if cib__op_attr_local is set.
 501          *
 502          * @COMPAT: Currently host is ignored. At a compatibility break, throw
 503          * an error (from cib_process_request() or earlier) if host is not NULL or
 504          * OUR_NODENAME.
 505          */
 506         crm_trace("Processing always-local %s op from client %s",
 507                   op, pcmk__client_name(cib_client));
 508 
 509         if (!pcmk__str_eq(host, OUR_NODENAME,
 510                           pcmk__str_casei|pcmk__str_null_matches)) {
 511 
 512             crm_warn("Operation '%s' is always local but its target host is "
 513                      "set to '%s'",
 514                      op, host);
 515         }
 516         return;
 517     }
 518 
 519     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 520         || !pcmk__str_eq(host, OUR_NODENAME,
 521                          pcmk__str_casei|pcmk__str_null_matches)) {
 522 
 523         // Forward modifying and non-local requests via cluster
 524         *process = FALSE;
 525         *needs_reply = FALSE;
 526         *local_notify = FALSE;
 527         *needs_forward = TRUE;
 528 
 529         crm_trace("%s op from %s needs to be forwarded to %s",
 530                   op, pcmk__client_name(cib_client),
 531                   pcmk__s(host, "all nodes"));
 532         return;
 533     }
 534 
 535     if (stand_alone) {
 536         crm_trace("Processing %s op from client %s (stand-alone)",
 537                   op, pcmk__client_name(cib_client));
 538 
 539     } else {
 540         crm_trace("Processing %saddressed %s op from client %s",
 541                   ((host != NULL)? "locally " : "un"),
 542                   op, pcmk__client_name(cib_client));
 543     }
 544 }
 545 
 546 static gboolean
 547 parse_peer_options(const cib__operation_t *operation, xmlNode *request,
     /*  */
 548                    gboolean *local_notify, gboolean *needs_reply,
 549                    gboolean *process)
 550 {
 551     /* TODO: What happens when an update comes in after node A
 552      * requests the CIB from node B, but before it gets the reply (and
 553      * sends out the replace operation)?
 554      *
 555      * (This may no longer be relevant since legacy mode was dropped; need to
 556      * trace code more closely to check.)
 557      */
 558     const char *host = NULL;
 559     const char *delegated = crm_element_value(request,
 560                                               PCMK__XA_CIB_DELEGATED_FROM);
 561     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 562     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 563     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 564 
 565     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 566 
 567     if (originator == NULL) { // Shouldn't be possible
 568         originator = "peer";
 569     }
 570 
 571     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
 572         // sync_our_cib() sets PCMK__XA_CIB_ISREPLYTO
 573         if (reply_to) {
 574             delegated = reply_to;
 575         }
 576         goto skip_is_reply;
 577 
 578     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
 579                             pcmk__str_none)) {
 580         // Nothing to do
 581 
 582     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 583         process_ping_reply(request);
 584         return FALSE;
 585 
 586     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
 587         /* Only the DC (node with the oldest software) should process
 588          * this operation if PCMK__XA_CIB_SCHEMA_MAX is unset.
 589          *
 590          * If the DC is happy it will then send out another
 591          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
 592          * upgrade.
 593          *
 594          * Except this time PCMK__XA_CIB_SCHEMA_MAX will be set which puts a
 595          * limit on how far newer nodes will go
 596          */
 597         const char *max = crm_element_value(request, PCMK__XA_CIB_SCHEMA_MAX);
 598         const char *upgrade_rc = crm_element_value(request,
 599                                                    PCMK__XA_CIB_UPGRADE_RC);
 600 
 601         crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s",
 602                   (is_reply? "reply" : "request"),
 603                   (based_is_primary? "primary" : "secondary"),
 604                   pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none"));
 605 
 606         if (upgrade_rc != NULL) {
 607             // Our upgrade request was rejected by DC, notify clients of result
 608             crm_xml_add(request, PCMK__XA_CIB_RC, upgrade_rc);
 609 
 610         } else if ((max == NULL) && based_is_primary) {
 611             /* We are the DC, check if this upgrade is allowed */
 612             goto skip_is_reply;
 613 
 614         } else if(max) {
 615             /* Ok, go ahead and upgrade to 'max' */
 616             goto skip_is_reply;
 617 
 618         } else {
 619             // Ignore broadcast client requests when we're not primary
 620             return FALSE;
 621         }
 622 
 623     } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 624         crm_info("Detected legacy %s global update from %s", op, originator);
 625         send_sync_request(NULL);
 626         return FALSE;
 627 
 628     } else if (is_reply
 629                && pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
 630         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 631         return FALSE;
 632 
 633     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 634         *local_notify = FALSE;
 635         if (reply_to == NULL) {
 636             *process = TRUE;
 637         } else { // Not possible?
 638             crm_debug("Ignoring shutdown request from %s because reply_to=%s",
 639                       originator, reply_to);
 640         }
 641         return *process;
 642     }
 643 
 644     if (is_reply) {
 645         crm_trace("Will notify local clients for %s reply from %s",
 646                   op, originator);
 647         *process = FALSE;
 648         *needs_reply = FALSE;
 649         *local_notify = TRUE;
 650         return TRUE;
 651     }
 652 
 653   skip_is_reply:
 654     *process = TRUE;
 655     *needs_reply = FALSE;
 656 
 657     *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
 658 
 659     host = crm_element_value(request, PCMK__XA_CIB_HOST);
 660     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 661         crm_trace("Processing %s request sent to us from %s", op, originator);
 662         *needs_reply = TRUE;
 663         return TRUE;
 664 
 665     } else if (host != NULL) {
 666         crm_trace("Ignoring %s request intended for CIB manager on %s",
 667                   op, host);
 668         return FALSE;
 669 
 670     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 671         *needs_reply = TRUE;
 672     }
 673 
 674     crm_trace("Processing %s request broadcast by %s call %s on %s "
 675               "(local clients will%s be notified)", op,
 676               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CLIENTNAME),
 677                       "client"),
 678               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CALLID),
 679                       "without ID"),
 680               originator, (*local_notify? "" : "not"));
 681     return TRUE;
 682 }
 683 
 684 /*!
 685  * \internal
 686  * \brief Forward a CIB request to the appropriate target host(s)
 687  *
 688  * \param[in] request  CIB request to forward
 689  */
 690 static void
 691 forward_request(xmlNode *request)
     /*  */
 692 {
 693     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 694     const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
 695     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
 696     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 697     const char *client_name = crm_element_value(request,
 698                                                 PCMK__XA_CIB_CLIENTNAME);
 699     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
 700     pcmk__node_status_t *peer = NULL;
 701 
 702     int log_level = LOG_INFO;
 703 
 704     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
 705         log_level = LOG_DEBUG;
 706     }
 707 
 708     do_crm_log(log_level,
 709                "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
 710                pcmk__s(op, "invalid"),
 711                pcmk__s(section, "all"),
 712                pcmk__s(host, "all"),
 713                pcmk__s(originator, "local"),
 714                pcmk__s(client_name, "unspecified"),
 715                pcmk__s(call_id, "unspecified"));
 716 
 717     crm_xml_add(request, PCMK__XA_CIB_DELEGATED_FROM, OUR_NODENAME);
 718 
 719     if (host != NULL) {
 720         peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster_member);
 721     }
 722     pcmk__cluster_send_message(peer, pcmk_ipc_based, request);
 723 
 724     // Return the request to its original state
 725     pcmk__xe_remove_attr(request, PCMK__XA_CIB_DELEGATED_FROM);
 726 }
 727 
 728 static void
 729 send_peer_reply(xmlNode *msg, const char *originator)
     /*  */
 730 {
 731     const pcmk__node_status_t *node = NULL;
 732 
 733     if ((msg == NULL) || (originator == NULL)) {
 734         return;
 735     }
 736 
 737     // Send reply via cluster to originating node
 738     node = pcmk__get_node(0, originator, NULL,
 739                           pcmk__node_search_cluster_member);
 740 
 741     crm_trace("Sending request result to %s only", originator);
 742     crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator);
 743     pcmk__cluster_send_message(node, pcmk_ipc_based, msg);
 744 }
 745 
 746 /*!
 747  * \internal
 748  * \brief Handle an IPC or CPG message containing a request
 749  *
 750  * \param[in,out] request        Request XML
 751  * \param[in] privileged         Whether privileged commands may be run
 752  *                               (see cib_server_ops[] definition)
 753  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
 754  *
 755  * \return Legacy Pacemaker return code
 756  */
 757 int
 758 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
 759                     const pcmk__client_t *cib_client)
 760 {
 761     // @TODO: Break into multiple smaller functions
 762     uint32_t call_options = cib_none;
 763 
 764     gboolean process = TRUE;        // Whether to process request locally now
 765     gboolean is_update = TRUE;      // Whether request would modify CIB
 766     gboolean needs_reply = TRUE;    // Whether to build a reply
 767     gboolean local_notify = FALSE;  // Whether to notify (local) requester
 768     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 769 
 770     xmlNode *op_reply = NULL;
 771     xmlNode *result_diff = NULL;
 772 
 773     int rc = pcmk_ok;
 774     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 775     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 776     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
 777     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
 778     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
 779     const char *client_name = crm_element_value(request,
 780                                                 PCMK__XA_CIB_CLIENTNAME);
 781     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 782 
 783     const cib__operation_t *operation = NULL;
 784     cib__op_fn_t op_function = NULL;
 785 
 786     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
 787                             cib_none);
 788     if (rc != pcmk_rc_ok) {
 789         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
 790     }
 791 
 792     if ((host != NULL) && (*host == '\0')) {
 793         host = NULL;
 794     }
 795 
 796     if (cib_client == NULL) {
 797         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
 798                   op, pcmk__s(client_name, "client"), call_id, originator,
 799                   pcmk__s(host, "all"), reply_to);
 800     } else {
 801         crm_xml_add(request, PCMK__XA_SRC, OUR_NODENAME);
 802         crm_trace("Processing local %s operation from %s/%s intended for %s",
 803                   op, pcmk__s(client_name, "client"), call_id,
 804                   pcmk__s(host, "all"));
 805     }
 806 
 807     rc = cib__get_operation(op, &operation);
 808     rc = pcmk_rc2legacy(rc);
 809     if (rc != pcmk_ok) {
 810         /* TODO: construct error reply? */
 811         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
 812         return rc;
 813     }
 814 
 815     op_function = based_get_op_function(operation);
 816     if (op_function == NULL) {
 817         crm_err("Operation %s not supported by CIB manager", op);
 818         return -EOPNOTSUPP;
 819     }
 820 
 821     if (cib_client != NULL) {
 822         parse_local_options(cib_client, operation, host, op,
 823                             &local_notify, &needs_reply, &process,
 824                             &needs_forward);
 825 
 826     } else if (!parse_peer_options(operation, request, &local_notify,
 827                                    &needs_reply, &process)) {
 828         return rc;
 829     }
 830 
 831     if (pcmk_is_set(call_options, cib_transaction)) {
 832         /* All requests in a transaction are processed locally against a working
 833          * CIB copy, and we don't notify for individual requests because the
 834          * entire transaction is atomic.
 835          *
 836          * We still call the option parser functions above, for the sake of log
 837          * messages and checking whether we're the target for peer requests.
 838          */
 839         process = TRUE;
 840         needs_reply = FALSE;
 841         local_notify = FALSE;
 842         needs_forward = FALSE;
 843     }
 844 
 845     is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies);
 846 
 847     if (pcmk_is_set(call_options, cib_discard_reply)) {
 848         /* If the request will modify the CIB, and we are in legacy mode, we
 849          * need to build a reply so we can broadcast a diff, even if the
 850          * requester doesn't want one.
 851          */
 852         needs_reply = FALSE;
 853         local_notify = FALSE;
 854         crm_trace("Client is not interested in the reply");
 855     }
 856 
 857     if (needs_forward) {
 858         forward_request(request);
 859         return rc;
 860     }
 861 
 862     if (cib_status != pcmk_ok) {
 863         rc = cib_status;
 864         crm_err("Operation ignored, cluster configuration is invalid."
 865                 " Please repair and restart: %s", pcmk_strerror(cib_status));
 866 
 867         op_reply = create_cib_reply(op, call_id, client_id, call_options, rc,
 868                                     the_cib);
 869 
 870     } else if (process) {
 871         time_t finished = 0;
 872         time_t now = time(NULL);
 873         int level = LOG_INFO;
 874         const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
 875         const char *admin_epoch_s = NULL;
 876         const char *epoch_s = NULL;
 877         const char *num_updates_s = NULL;
 878 
 879         rc = cib_process_command(request, operation, op_function, &op_reply,
 880                                  &result_diff, privileged);
 881 
 882         if (!is_update) {
 883             level = LOG_TRACE;
 884 
 885         } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 886             switch (rc) {
 887                 case pcmk_ok:
 888                     level = LOG_INFO;
 889                     break;
 890                 case -pcmk_err_old_data:
 891                 case -pcmk_err_diff_resync:
 892                 case -pcmk_err_diff_failed:
 893                     level = LOG_TRACE;
 894                     break;
 895                 default:
 896                     level = LOG_ERR;
 897             }
 898 
 899         } else if (rc != pcmk_ok) {
 900             level = LOG_WARNING;
 901         }
 902 
 903         if (the_cib != NULL) {
 904             admin_epoch_s = crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH);
 905             epoch_s = crm_element_value(the_cib, PCMK_XA_EPOCH);
 906             num_updates_s = crm_element_value(the_cib, PCMK_XA_NUM_UPDATES);
 907         }
 908 
 909         do_crm_log(level,
 910                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
 911                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
 912                    originator ? originator : "local",
 913                    pcmk__s(client_name, "client"), call_id,
 914                    pcmk__s(admin_epoch_s, "0"),
 915                    pcmk__s(epoch_s, "0"),
 916                    pcmk__s(num_updates_s, "0"));
 917 
 918         finished = time(NULL);
 919         if ((finished - now) > 3) {
 920             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
 921             crm_write_blackbox(0, NULL);
 922         }
 923 
 924         if (op_reply == NULL && (needs_reply || local_notify)) {
 925             crm_err("Unexpected NULL reply to message");
 926             crm_log_xml_err(request, "null reply");
 927             needs_reply = FALSE;
 928             local_notify = FALSE;
 929         }
 930     }
 931 
 932     if (is_update) {
 933         crm_trace("Completed pre-sync update from %s/%s/%s%s",
 934                   originator ? originator : "local",
 935                   pcmk__s(client_name, "client"), call_id,
 936                   local_notify?" with local notification":"");
 937 
 938     } else if (!needs_reply || stand_alone) {
 939         // This was a non-originating secondary update
 940         crm_trace("Completed update as secondary");
 941 
 942     } else if ((cib_client == NULL)
 943                && !pcmk_is_set(call_options, cib_discard_reply)) {
 944 
 945         if (is_update == FALSE || result_diff == NULL) {
 946             crm_trace("Request not broadcast: R/O call");
 947 
 948         } else if (rc != pcmk_ok) {
 949             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
 950 
 951         } else {
 952             crm_trace("Directing reply to %s", originator);
 953         }
 954 
 955         send_peer_reply(op_reply, originator);
 956     }
 957 
 958     if (local_notify && client_id) {
 959         crm_trace("Performing local %ssync notification for %s",
 960                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
 961                   client_id);
 962         if (process == FALSE) {
 963             do_local_notify(request, client_id,
 964                             pcmk_is_set(call_options, cib_sync_call),
 965                             (cib_client == NULL));
 966         } else {
 967             do_local_notify(op_reply, client_id,
 968                             pcmk_is_set(call_options, cib_sync_call),
 969                             (cib_client == NULL));
 970         }
 971     }
 972 
 973     pcmk__xml_free(op_reply);
 974     pcmk__xml_free(result_diff);
 975 
 976     return rc;
 977 }
 978 
 979 /*!
 980  * \internal
 981  * \brief Get a CIB operation's input from the request XML
 982  *
 983  * \param[in]  request  CIB request XML
 984  * \param[in]  type     CIB operation type
 985  * \param[out] section  Where to store CIB section name
 986  *
 987  * \return Input XML for CIB operation
 988  *
 989  * \note If not \c NULL, the return value is a non-const pointer to part of
 990  *       \p request. The caller should not free it directly.
 991  */
 992 static xmlNode *
 993 prepare_input(const xmlNode *request, enum cib__op_type type,
     /*  */
 994               const char **section)
 995 {
 996     xmlNode *wrapper = pcmk__xe_first_child(request, PCMK__XE_CIB_CALLDATA,
 997                                             NULL, NULL);
 998     xmlNode *input = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 999 
1000     if (type == cib__op_apply_patch) {
1001         *section = NULL;
1002     } else {
1003         *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
1004     }
1005 
1006     // Grab the specified section
1007     if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) {
1008         input = pcmk_find_cib_element(input, *section);
1009     }
1010 
1011     return input;
1012 }
1013 
1014 #define XPATH_CONFIG_CHANGE         \
1015     "//" PCMK_XE_CHANGE             \
1016     "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]"
1017 
1018 static bool
1019 contains_config_change(xmlNode *diff)
     /*  */
1020 {
1021     bool changed = false;
1022 
1023     if (diff) {
1024         xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
1025 
1026         if (numXpathResults(xpathObj) > 0) {
1027             changed = true;
1028         }
1029         freeXpathObject(xpathObj);
1030     }
1031     return changed;
1032 }
1033 
1034 static int
1035 cib_process_command(xmlNode *request, const cib__operation_t *operation,
     /*  */
1036                     cib__op_fn_t op_function, xmlNode **reply,
1037                     xmlNode **cib_diff, bool privileged)
1038 {
1039     xmlNode *input = NULL;
1040     xmlNode *output = NULL;
1041     xmlNode *result_cib = NULL;
1042 
1043     uint32_t call_options = cib_none;
1044 
1045     const char *op = NULL;
1046     const char *section = NULL;
1047     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
1048     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
1049     const char *client_name = crm_element_value(request,
1050                                                 PCMK__XA_CIB_CLIENTNAME);
1051     const char *originator = crm_element_value(request, PCMK__XA_SRC);
1052 
1053     int rc = pcmk_ok;
1054 
1055     bool config_changed = false;
1056     bool manage_counters = true;
1057 
1058     static mainloop_timer_t *digest_timer = NULL;
1059 
1060     pcmk__assert(cib_status == pcmk_ok);
1061 
1062     if(digest_timer == NULL) {
1063         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1064     }
1065 
1066     *reply = NULL;
1067     *cib_diff = NULL;
1068 
1069     /* Start processing the request... */
1070     op = crm_element_value(request, PCMK__XA_CIB_OP);
1071     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
1072                             cib_none);
1073     if (rc != pcmk_rc_ok) {
1074         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
1075     }
1076 
1077     if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) {
1078         rc = -EACCES;
1079         crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc));
1080         goto done;
1081     }
1082 
1083     input = prepare_input(request, operation->type, &section);
1084 
1085     if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
1086         rc = cib_perform_op(NULL, op, call_options, op_function, true, section,
1087                             request, input, false, &config_changed, &the_cib,
1088                             &result_cib, NULL, &output);
1089 
1090         CRM_CHECK(result_cib == NULL, pcmk__xml_free(result_cib));
1091         goto done;
1092     }
1093 
1094     /* @COMPAT: Handle a valid write action (legacy)
1095      *
1096      * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff
1097      * portion is. However, PCMK__XA_CIB_UPDATE may be set by a sync operation
1098      * even in non-legacy mode, and manage_counters tells xml_create_patchset()
1099      * whether to update version/epoch info.
1100      */
1101     if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
1102         manage_counters = false;
1103         cib__set_call_options(call_options, "call", cib_force_diff);
1104         crm_trace("Global update detected");
1105 
1106         CRM_LOG_ASSERT(pcmk__str_any_of(op,
1107                                         PCMK__CIB_REQUEST_APPLY_PATCH,
1108                                         PCMK__CIB_REQUEST_REPLACE,
1109                                         NULL));
1110     }
1111 
1112     ping_modified_since = TRUE;
1113 
1114     // result_cib must not be modified after cib_perform_op() returns
1115     rc = cib_perform_op(NULL, op, call_options, op_function, false, section,
1116                         request, input, manage_counters, &config_changed,
1117                         &the_cib, &result_cib, cib_diff, &output);
1118 
1119     /* Always write to disk for successful ops with the flag set. This also
1120      * negates the need to detect ordering changes.
1121      */
1122     if ((rc == pcmk_ok)
1123         && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1124 
1125         config_changed = true;
1126     }
1127 
1128     if ((rc == pcmk_ok)
1129         && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) {
1130 
1131         if (result_cib != the_cib) {
1132             if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1133                 config_changed = true;
1134             }
1135 
1136             crm_trace("Activating %s->%s%s",
1137                       crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
1138                       crm_element_value(result_cib, PCMK_XA_NUM_UPDATES),
1139                       (config_changed? " changed" : ""));
1140 
1141             rc = activateCibXml(result_cib, config_changed, op);
1142             if (rc != pcmk_ok) {
1143                 crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc));
1144             }
1145         }
1146 
1147         if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
1148             cib_read_config(config_hash, result_cib);
1149         }
1150 
1151         /* @COMPAT Nodes older than feature set 3.19.0 don't support
1152          * transactions. In a mixed-version cluster with nodes <3.19.0, we must
1153          * sync the updated CIB, so that the older nodes receive the changes.
1154          * Any node that has already applied the transaction will ignore the
1155          * synced CIB.
1156          *
1157          * To ensure the updated CIB is synced from only one node, we sync it
1158          * from the originator.
1159          */
1160         if ((operation->type == cib__op_commit_transact)
1161             && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei)
1162             && compare_version(crm_element_value(the_cib,
1163                                                  PCMK_XA_CRM_FEATURE_SET),
1164                                "3.19.0") < 0) {
1165 
1166             sync_our_cib(request, TRUE);
1167         }
1168 
1169         mainloop_timer_stop(digest_timer);
1170         mainloop_timer_start(digest_timer);
1171 
1172     } else if (rc == -pcmk_err_schema_validation) {
1173         pcmk__assert(result_cib != the_cib);
1174 
1175         if (output != NULL) {
1176             crm_log_xml_info(output, "cib:output");
1177             pcmk__xml_free(output);
1178         }
1179 
1180         output = result_cib;
1181 
1182     } else {
1183         crm_trace("Not activating %d %d %s", rc,
1184                   pcmk_is_set(call_options, cib_dryrun),
1185                   crm_element_value(result_cib, PCMK_XA_NUM_UPDATES));
1186 
1187         if (result_cib != the_cib) {
1188             pcmk__xml_free(result_cib);
1189         }
1190     }
1191 
1192     if (!pcmk_any_flags_set(call_options,
1193                             cib_dryrun|cib_inhibit_notify|cib_transaction)) {
1194         crm_trace("Sending notifications %d",
1195                   pcmk_is_set(call_options, cib_dryrun));
1196         cib_diff_notify(op, rc, call_id, client_id, client_name, originator,
1197                         input, *cib_diff);
1198     }
1199 
1200     pcmk__log_xml_patchset(LOG_TRACE, *cib_diff);
1201 
1202   done:
1203     if (!pcmk_is_set(call_options, cib_discard_reply)) {
1204         *reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1205                                   output);
1206     }
1207 
1208     if (output != the_cib) {
1209         pcmk__xml_free(output);
1210     }
1211     crm_trace("done");
1212     return rc;
1213 }
1214 
1215 void
1216 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1217 {
1218     const char *reason = NULL;
1219     const char *originator = crm_element_value(msg, PCMK__XA_SRC);
1220 
1221     if (pcmk__peer_cache == NULL) {
1222         reason = "membership not established";
1223         goto bail;
1224     }
1225 
1226     if (crm_element_value(msg, PCMK__XA_CIB_CLIENTNAME) == NULL) {
1227         crm_xml_add(msg, PCMK__XA_CIB_CLIENTNAME, originator);
1228     }
1229 
1230     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1231     cib_process_request(msg, TRUE, NULL);
1232     return;
1233 
1234   bail:
1235     if (reason) {
1236         const char *op = crm_element_value(msg, PCMK__XA_CIB_OP);
1237 
1238         crm_warn("Discarding %s message from %s: %s", op, originator, reason);
1239     }
1240 }
1241 
1242 static gboolean
1243 cib_force_exit(gpointer data)
     /*  */
1244 {
1245     crm_notice("Forcing exit!");
1246     terminate_cib(__func__, CRM_EX_ERROR);
1247     return FALSE;
1248 }
1249 
1250 static void
1251 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1252 {
1253     pcmk__client_t *a_client = value;
1254 
1255     crm_err("Can't disconnect client %s: Not implemented",
1256             pcmk__client_name(a_client));
1257 }
1258 
1259 static void
1260 initiate_exit(void)
     /*  */
1261 {
1262     int active = 0;
1263     xmlNode *leaving = NULL;
1264 
1265     active = pcmk__cluster_num_active_nodes();
1266     if (active < 2) { // This is the last active node
1267         terminate_cib(__func__, 0);
1268         return;
1269     }
1270 
1271     crm_info("Sending shutdown request to %d peers", active);
1272 
1273     leaving = pcmk__xe_create(NULL, PCMK__XE_EXIT_NOTIFICATION);
1274     crm_xml_add(leaving, PCMK__XA_T, PCMK__VALUE_CIB);
1275     crm_xml_add(leaving, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_SHUTDOWN);
1276 
1277     pcmk__cluster_send_message(NULL, pcmk_ipc_based, leaving);
1278     pcmk__xml_free(leaving);
1279 
1280     pcmk__create_timer(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1281 }
1282 
1283 void
1284 cib_shutdown(int nsig)
     /*  */
1285 {
1286     struct qb_ipcs_stats srv_stats;
1287 
1288     if (cib_shutdown_flag == FALSE) {
1289         int disconnects = 0;
1290         qb_ipcs_connection_t *c = NULL;
1291 
1292         cib_shutdown_flag = TRUE;
1293 
1294         c = qb_ipcs_connection_first_get(ipcs_rw);
1295         while (c != NULL) {
1296             qb_ipcs_connection_t *last = c;
1297 
1298             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1299 
1300             crm_debug("Disconnecting r/w client %p...", last);
1301             qb_ipcs_disconnect(last);
1302             qb_ipcs_connection_unref(last);
1303             disconnects++;
1304         }
1305 
1306         c = qb_ipcs_connection_first_get(ipcs_ro);
1307         while (c != NULL) {
1308             qb_ipcs_connection_t *last = c;
1309 
1310             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1311 
1312             crm_debug("Disconnecting r/o client %p...", last);
1313             qb_ipcs_disconnect(last);
1314             qb_ipcs_connection_unref(last);
1315             disconnects++;
1316         }
1317 
1318         c = qb_ipcs_connection_first_get(ipcs_shm);
1319         while (c != NULL) {
1320             qb_ipcs_connection_t *last = c;
1321 
1322             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1323 
1324             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1325             qb_ipcs_disconnect(last);
1326             qb_ipcs_connection_unref(last);
1327             disconnects++;
1328         }
1329 
1330         disconnects += pcmk__ipc_client_count();
1331 
1332         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1333         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1334         crm_info("Disconnected %d clients", disconnects);
1335     }
1336 
1337     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1338 
1339     if (pcmk__ipc_client_count() == 0) {
1340         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1341         initiate_exit();
1342 
1343     } else {
1344         crm_info("Waiting on %d clients to disconnect (%d)",
1345                  pcmk__ipc_client_count(), srv_stats.active_connections);
1346     }
1347 }
1348 
1349 extern int remote_fd;
1350 extern int remote_tls_fd;
1351 
1352 /*!
1353  * \internal
1354  * \brief Close remote sockets, free the global CIB and quit
1355  *
1356  * \param[in] caller           Name of calling function (for log message)
1357  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1358  */
1359 void
1360 terminate_cib(const char *caller, int fast)
     /*  */
1361 {
1362     crm_info("%s: Exiting%s...", caller,
1363              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1364 
1365     if (remote_fd > 0) {
1366         close(remote_fd);
1367         remote_fd = 0;
1368     }
1369     if (remote_tls_fd > 0) {
1370         close(remote_tls_fd);
1371         remote_tls_fd = 0;
1372     }
1373 
1374     uninitializeCib();
1375 
1376     if (fast > 0) {
1377         /* Quit fast on error */
1378         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1379         crm_exit(fast);
1380 
1381     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1382         /* Quit via returning from the main loop. If fast == -1, we skip the
1383          * disconnect here, and it will be done when the main loop returns
1384          * (this allows the peer status callback to avoid messing with the
1385          * peer caches).
1386          */
1387         if (fast == 0) {
1388             pcmk_cluster_disconnect(crm_cluster);
1389         }
1390         g_main_loop_quit(mainloop);
1391 
1392     } else {
1393         /* Quit via clean exit. Even the peer status callback can disconnect
1394          * here, because we're not returning control to the caller. */
1395         pcmk_cluster_disconnect(crm_cluster);
1396         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1397         crm_exit(CRM_EX_OK);
1398     }
1399 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
