daemons/based/based

/* */
This source file includes following definitions.
cib_legacy_mode
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
do_local_notify
local_notify_destroy_callback
check_local_notify
queue_local_notify
parse_local_options_v1
parse_local_options_v2
parse_local_options
parse_peer_options_v1
parse_peer_options_v2
parse_peer_options
forward_request
send_peer_reply
cib_process_request
calculate_section_digest
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
cib_shutdown
initiate_exit
terminate_cib
   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>   // PRIu64
  22 
  23 #include <crm/crm.h>
  24 #include <crm/cib.h>
  25 #include <crm/msg_xml.h>
  26 #include <crm/cluster/internal.h>
  27 
  28 #include <crm/common/xml.h>
  29 #include <crm/common/remote_internal.h>
  30 
  31 #include <pacemaker-based.h>
  32 
  33 #define EXIT_ESCALATION_MS 10000
  34 
  35 static unsigned long cib_local_bcast_num = 0;
  36 
  37 typedef struct cib_local_notify_s {
  38     xmlNode *notify_src;
  39     char *client_id;
  40     gboolean from_peer;
  41     gboolean sync_reply;
  42 } cib_local_notify_t;
  43 
  44 int next_client_id = 0;
  45 
  46 gboolean legacy_mode = FALSE;
  47 
  48 qb_ipcs_service_t *ipcs_ro = NULL;
  49 qb_ipcs_service_t *ipcs_rw = NULL;
  50 qb_ipcs_service_t *ipcs_shm = NULL;
  51 
  52 void send_cib_replace(const xmlNode * sync_request, const char *host);
  53 static void cib_process_request(xmlNode *request, gboolean privileged,
  54                                 const pcmk__client_t *cib_client);
  55 
  56 
  57 static int cib_process_command(xmlNode *request, xmlNode **reply,
  58                                xmlNode **cib_diff, gboolean privileged);
  59 
  60 gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
  61                              gboolean privileged);
  62 
  63 gboolean cib_legacy_mode(void)
     /*  */
  64 {
  65     return legacy_mode;
  66 }
  67 
  68 
  69 static int32_t
  70 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  71 {
  72     if (cib_shutdown_flag) {
  73         crm_info("Ignoring new IPC client [%d] during shutdown",
  74                  pcmk__client_pid(c));
  75         return -EPERM;
  76     }
  77 
  78     if (pcmk__new_client(c, uid, gid) == NULL) {
  79         return -EIO;
  80     }
  81     return 0;
  82 }
  83 
  84 static int32_t
  85 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  86 {
  87     pcmk__client_t *client = pcmk__find_client(c);
  88 
  89     crm_trace("%p message from %s", c, client->id);
  90     return cib_common_callback(c, data, size, TRUE);
  91 }
  92 
  93 static int32_t
  94 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  95 {
  96     pcmk__client_t *client = pcmk__find_client(c);
  97 
  98     crm_trace("%p message from %s", c, client->id);
  99     return cib_common_callback(c, data, size, FALSE);
 100 }
 101 
 102 /* Error code means? */
 103 static int32_t
 104 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 105 {
 106     pcmk__client_t *client = pcmk__find_client(c);
 107 
 108     if (client == NULL) {
 109         return 0;
 110     }
 111     crm_trace("Connection %p", c);
 112     pcmk__free_client(client);
 113     return 0;
 114 }
 115 
 116 static void
 117 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 118 {
 119     crm_trace("Connection %p", c);
 120     cib_ipc_closed(c);
 121     if (cib_shutdown_flag) {
 122         cib_shutdown(0);
 123     }
 124 }
 125 
 126 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 127     .connection_accept = cib_ipc_accept,
 128     .connection_created = NULL,
 129     .msg_process = cib_ipc_dispatch_ro,
 130     .connection_closed = cib_ipc_closed,
 131     .connection_destroyed = cib_ipc_destroy
 132 };
 133 
 134 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 135     .connection_accept = cib_ipc_accept,
 136     .connection_created = NULL,
 137     .msg_process = cib_ipc_dispatch_rw,
 138     .connection_closed = cib_ipc_closed,
 139     .connection_destroyed = cib_ipc_destroy
 140 };
 141 
 142 void
 143 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 144                            pcmk__client_t *cib_client, gboolean privileged)
 145 {
 146     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 147 
 148     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 149         if (flags & crm_ipc_client_response) {
 150             xmlNode *ack = create_xml_node(NULL, __func__);
 151 
 152             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
 153             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
 154             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 155             cib_client->request_id = 0;
 156             free_xml(ack);
 157         }
 158         return;
 159 
 160     } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
 161         /* Update the notify filters for this client */
 162         int on_off = 0;
 163         crm_exit_t status = CRM_EX_OK;
 164         uint64_t bit = UINT64_C(0);
 165         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 166 
 167         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 168 
 169         crm_debug("Setting %s callbacks %s for client %s",
 170                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 171 
 172         if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
 173             bit = cib_notify_post;
 174 
 175         } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
 176             bit = cib_notify_pre;
 177 
 178         } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
 179             bit = cib_notify_confirm;
 180 
 181         } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
 182             bit = cib_notify_diff;
 183 
 184         } else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
 185             bit = cib_notify_replace;
 186 
 187         } else {
 188             status = CRM_EX_INVALID_PARAM;
 189         }
 190 
 191         if (bit != 0) {
 192             if (on_off) {
 193                 pcmk__set_client_flags(cib_client, bit);
 194             } else {
 195                 pcmk__clear_client_flags(cib_client, bit);
 196             }
 197         }
 198 
 199         pcmk__ipc_send_ack(cib_client, id, flags, "ack", NULL, status);
 200         return;
 201     }
 202 
 203     cib_process_request(op_request, privileged, cib_client);
 204 }
 205 
 206 int32_t
 207 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 208 {
 209     uint32_t id = 0;
 210     uint32_t flags = 0;
 211     int call_options = 0;
 212     pcmk__client_t *cib_client = pcmk__find_client(c);
 213     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 214 
 215     if (op_request) {
 216         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 217     }
 218 
 219     if (op_request == NULL) {
 220         crm_trace("Invalid message from %p", c);
 221         pcmk__ipc_send_ack(cib_client, id, flags, "nack", NULL, CRM_EX_PROTOCOL);
 222         return 0;
 223 
 224     } else if(cib_client == NULL) {
 225         crm_trace("Invalid client %p", c);
 226         return 0;
 227     }
 228 
 229     if (pcmk_is_set(call_options, cib_sync_call)) {
 230         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 231         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 232         cib_client->request_id = id;    /* Reply only to the last one */
 233     }
 234 
 235     if (cib_client->name == NULL) {
 236         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 237 
 238         if (value == NULL) {
 239             cib_client->name = pcmk__itoa(cib_client->pid);
 240         } else {
 241             cib_client->name = strdup(value);
 242             if (crm_is_daemon_name(value)) {
 243                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 244             }
 245         }
 246     }
 247 
 248     /* Allow cluster daemons more leeway before being evicted */
 249     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 250         const char *qmax = cib_config_lookup("cluster-ipc-limit");
 251 
 252         if (pcmk__set_client_queue_max(cib_client, qmax)) {
 253             crm_trace("IPC threshold for client %s[%u] is now %u",
 254                       pcmk__client_name(cib_client), cib_client->pid,
 255                       cib_client->queue_max);
 256         }
 257     }
 258 
 259     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
 260     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 261 
 262     CRM_LOG_ASSERT(cib_client->user != NULL);
 263     pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
 264 
 265     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 266     free_xml(op_request);
 267 
 268     return 0;
 269 }
 270 
 271 static uint64_t ping_seq = 0;
 272 static char *ping_digest = NULL;
 273 static bool ping_modified_since = FALSE;
 274 int sync_our_cib(xmlNode * request, gboolean all);
 275 
 276 static gboolean
 277 cib_digester_cb(gpointer data)
     /*  */
 278 {
 279     if (based_is_primary) {
 280         char buffer[32];
 281         xmlNode *ping = create_xml_node(NULL, "ping");
 282 
 283         ping_seq++;
 284         free(ping_digest);
 285         ping_digest = NULL;
 286         ping_modified_since = FALSE;
 287         snprintf(buffer, 32, "%" PRIu64, ping_seq);
 288         crm_trace("Requesting peer digests (%s)", buffer);
 289 
 290         crm_xml_add(ping, F_TYPE, "cib");
 291         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
 292         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 293 
 294         crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 295         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 296 
 297         free_xml(ping);
 298     }
 299     return FALSE;
 300 }
 301 
 302 static void
 303 process_ping_reply(xmlNode *reply) 
     /*  */
 304 {
 305     uint64_t seq = 0;
 306     const char *host = crm_element_value(reply, F_ORIG);
 307 
 308     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
 309     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
 310     const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
 311 
 312     if (seq_s == NULL) {
 313         crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
 314         return;
 315 
 316     } else {
 317         long long seq_ll;
 318 
 319         if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
 320             return;
 321         }
 322         seq = (uint64_t) seq_ll;
 323     }
 324 
 325     if(digest == NULL) {
 326         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 327 
 328     } else if(seq != ping_seq) {
 329         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 330 
 331     } else if(ping_modified_since) {
 332         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 333 
 334     } else {
 335         const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
 336 
 337         if(ping_digest == NULL) {
 338             crm_trace("Calculating new digest");
 339             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
 340         }
 341 
 342         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 343         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 344             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
 345 
 346             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 347                        crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
 348                        crm_element_value(the_cib, XML_ATTR_GENERATION),
 349                        crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
 350                        ping_digest, host,
 351                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
 352                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
 353                        remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
 354                        digest, remote_cib);
 355 
 356             if(remote_cib && remote_cib->children) {
 357                 /* Additional debug */
 358                 xml_calculate_changes(the_cib, remote_cib);
 359                 xml_log_changes(LOG_INFO, __func__, remote_cib);
 360                 crm_trace("End of differences");
 361             }
 362 
 363             free_xml(remote_cib);
 364             sync_our_cib(reply, FALSE);
 365         }
 366     }
 367 }
 368 
 369 static void
 370 do_local_notify(xmlNode * notify_src, const char *client_id,
     /*  */
 371                 gboolean sync_reply, gboolean from_peer)
 372 {
 373     int rid = 0;
 374     int call_id = 0;
 375     pcmk__client_t *client_obj = NULL;
 376 
 377     CRM_ASSERT(notify_src && client_id);
 378 
 379     crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
 380 
 381     client_obj = pcmk__find_client_by_id(client_id);
 382     if (client_obj == NULL) {
 383         crm_debug("Could not send response %d: client %s not found",
 384                   call_id, client_id);
 385         return;
 386     }
 387 
 388     if (sync_reply) {
 389         if (client_obj->ipcs) {
 390             CRM_LOG_ASSERT(client_obj->request_id);
 391 
 392             rid = client_obj->request_id;
 393             client_obj->request_id = 0;
 394 
 395             crm_trace("Sending response %d to client %s%s",
 396                       rid, pcmk__client_name(client_obj),
 397                       (from_peer? " (originator of delegated request)" : ""));
 398         } else {
 399             crm_trace("Sending response (call %d) to client %s%s",
 400                       call_id, pcmk__client_name(client_obj),
 401                       (from_peer? " (originator of delegated request)" : ""));
 402         }
 403 
 404     } else {
 405         crm_trace("Sending event %d to client %s%s",
 406                   call_id, pcmk__client_name(client_obj),
 407                   (from_peer? " (originator of delegated request)" : ""));
 408     }
 409 
 410     switch (PCMK__CLIENT_TYPE(client_obj)) {
 411         case pcmk__client_ipc:
 412             {
 413                 int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
 414                                             (sync_reply? crm_ipc_flags_none
 415                                              : crm_ipc_server_event));
 416 
 417                 if (rc != pcmk_rc_ok) {
 418                     crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
 419                              (sync_reply? "Synchronous" : "Asynchronous"),
 420                              pcmk__client_name(client_obj), pcmk_rc_str(rc),
 421                              rc);
 422                 }
 423             }
 424             break;
 425 #ifdef HAVE_GNUTLS_GNUTLS_H
 426         case pcmk__client_tls:
 427 #endif
 428         case pcmk__client_tcp:
 429             pcmk__remote_send_xml(client_obj->remote, notify_src);
 430             break;
 431         default:
 432             crm_err("Unknown transport for client %s "
 433                     CRM_XS " flags=%#016" PRIx64,
 434                     pcmk__client_name(client_obj), client_obj->flags);
 435     }
 436 }
 437 
 438 static void
 439 local_notify_destroy_callback(gpointer data)
     /*  */
 440 {
 441     cib_local_notify_t *notify = data;
 442 
 443     free_xml(notify->notify_src);
 444     free(notify->client_id);
 445     free(notify);
 446 }
 447 
 448 static void
 449 check_local_notify(int bcast_id)
     /*  */
 450 {
 451     cib_local_notify_t *notify = NULL;
 452 
 453     if (!local_notify_queue) {
 454         return;
 455     }
 456 
 457     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 458 
 459     if (notify) {
 460         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
 461                         notify->from_peer);
 462         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
 463     }
 464 }
 465 
 466 static void
 467 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
     /*  */
 468                    gboolean from_peer)
 469 {
 470     cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
 471 
 472     notify->notify_src = notify_src;
 473     notify->client_id = strdup(client_id);
 474     notify->sync_reply = sync_reply;
 475     notify->from_peer = from_peer;
 476 
 477     if (!local_notify_queue) {
 478         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
 479     }
 480     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
 481     // cppcheck doesn't know notify will get freed when hash table is destroyed
 482     // cppcheck-suppress memleak
 483 }
 484 
 485 static void
 486 parse_local_options_v1(const pcmk__client_t *cib_client, int call_type,
     /*  */
 487                        int call_options, const char *host, const char *op,
 488                        gboolean *local_notify, gboolean *needs_reply,
 489                        gboolean *process, gboolean *needs_forward)
 490 {
 491     if (cib_op_modifies(call_type)
 492         && !(call_options & cib_inhibit_bcast)) {
 493         /* we need to send an update anyway */
 494         *needs_reply = TRUE;
 495     } else {
 496         *needs_reply = FALSE;
 497     }
 498 
 499     if (host == NULL && (call_options & cib_scope_local)) {
 500         crm_trace("Processing locally scoped %s op from client %s",
 501                   op, pcmk__client_name(cib_client));
 502         *local_notify = TRUE;
 503 
 504     } else if ((host == NULL) && based_is_primary) {
 505         crm_trace("Processing %s op locally from client %s as primary",
 506                   op, pcmk__client_name(cib_client));
 507         *local_notify = TRUE;
 508 
 509     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 510         crm_trace("Processing locally addressed %s op from client %s",
 511                   op, pcmk__client_name(cib_client));
 512         *local_notify = TRUE;
 513 
 514     } else if (stand_alone) {
 515         *needs_forward = FALSE;
 516         *local_notify = TRUE;
 517         *process = TRUE;
 518 
 519     } else {
 520         crm_trace("%s op from %s needs to be forwarded to client %s",
 521                   op, pcmk__client_name(cib_client),
 522                   pcmk__s(host, "the primary instance"));
 523         *needs_forward = TRUE;
 524         *process = FALSE;
 525     }
 526 }
 527 
 528 static void
 529 parse_local_options_v2(const pcmk__client_t *cib_client, int call_type,
     /*  */
 530                        int call_options, const char *host, const char *op,
 531                        gboolean *local_notify, gboolean *needs_reply,
 532                        gboolean *process, gboolean *needs_forward)
 533 {
 534     if (cib_op_modifies(call_type)) {
 535         if (pcmk__str_any_of(op, PCMK__CIB_REQUEST_PRIMARY,
 536                              PCMK__CIB_REQUEST_SECONDARY, NULL)) {
 537             /* Always handle these locally */
 538             *process = TRUE;
 539             *needs_reply = FALSE;
 540             *local_notify = TRUE;
 541             *needs_forward = FALSE;
 542             return;
 543 
 544         } else {
 545             /* Redirect all other updates via CPG */
 546             *needs_reply = TRUE;
 547             *needs_forward = TRUE;
 548             *process = FALSE;
 549             crm_trace("%s op from %s needs to be forwarded to client %s",
 550                       op, pcmk__client_name(cib_client),
 551                       pcmk__s(host, "the primary instance"));
 552             return;
 553         }
 554     }
 555 
 556 
 557     *process = TRUE;
 558     *needs_reply = FALSE;
 559     *local_notify = TRUE;
 560     *needs_forward = FALSE;
 561 
 562     if (stand_alone) {
 563         crm_trace("Processing %s op from client %s (stand-alone)",
 564                   op, pcmk__client_name(cib_client));
 565 
 566     } else if (host == NULL) {
 567         crm_trace("Processing unaddressed %s op from client %s",
 568                   op, pcmk__client_name(cib_client));
 569 
 570     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 571         crm_trace("Processing locally addressed %s op from client %s",
 572                   op, pcmk__client_name(cib_client));
 573 
 574     } else {
 575         crm_trace("%s op from %s needs to be forwarded to client %s",
 576                   op, pcmk__client_name(cib_client), host);
 577         *needs_forward = TRUE;
 578         *process = FALSE;
 579     }
 580 }
 581 
 582 static void
 583 parse_local_options(const pcmk__client_t *cib_client, int call_type,
     /*  */
 584                     int call_options, const char *host, const char *op,
 585                     gboolean *local_notify, gboolean *needs_reply,
 586                     gboolean *process, gboolean *needs_forward)
 587 {
 588     if(cib_legacy_mode()) {
 589         parse_local_options_v1(cib_client, call_type, call_options, host,
 590                                op, local_notify, needs_reply, process, needs_forward);
 591     } else {
 592         parse_local_options_v2(cib_client, call_type, call_options, host,
 593                                op, local_notify, needs_reply, process, needs_forward);
 594     }
 595 }
 596 
 597 static gboolean
 598 parse_peer_options_v1(int call_type, xmlNode * request,
     /*  */
 599                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 600                    gboolean * needs_forward)
 601 {
 602     const char *op = NULL;
 603     const char *host = NULL;
 604     const char *delegated = NULL;
 605     const char *originator = crm_element_value(request, F_ORIG);
 606     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 607 
 608     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 609 
 610     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 611         *needs_reply = FALSE;
 612         if (is_reply) {
 613             *local_notify = TRUE;
 614             crm_trace("Processing global/peer update from %s"
 615                       " that originated from us", originator);
 616         } else {
 617             crm_trace("Processing global/peer update from %s", originator);
 618         }
 619         return TRUE;
 620     }
 621 
 622     op = crm_element_value(request, F_CIB_OPERATION);
 623     crm_trace("Processing %s request sent by %s", op, originator);
 624     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 625         /* Always process these */
 626         *local_notify = FALSE;
 627         if (reply_to == NULL || is_reply) {
 628             *process = TRUE;
 629         }
 630         if (is_reply) {
 631             *needs_reply = FALSE;
 632         }
 633         return *process;
 634     }
 635 
 636     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 637         process_ping_reply(request);
 638         return FALSE;
 639     }
 640 
 641     if (is_reply) {
 642         crm_trace("Forward reply sent from %s to local clients", originator);
 643         *process = FALSE;
 644         *needs_reply = FALSE;
 645         *local_notify = TRUE;
 646         return TRUE;
 647     }
 648 
 649     host = crm_element_value(request, F_CIB_HOST);
 650     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 651         crm_trace("Processing %s request sent to us from %s", op, originator);
 652         return TRUE;
 653 
 654     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 655         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
 656         *needs_reply = TRUE;
 657         return TRUE;
 658 
 659     } else if ((host == NULL) && based_is_primary) {
 660         crm_trace("Processing %s request sent to primary instance from %s",
 661                   op, originator);
 662         return TRUE;
 663     }
 664 
 665     delegated = crm_element_value(request, F_CIB_DELEGATED);
 666     if (delegated != NULL) {
 667         crm_trace("Ignoring message for primary instance");
 668 
 669     } else if (host != NULL) {
 670         /* this is for a specific instance and we're not it */
 671         crm_trace("Ignoring msg for instance on %s", host);
 672 
 673     } else if ((reply_to == NULL) && !based_is_primary) {
 674         // This is for the primary instance, and we're not it
 675         crm_trace("Ignoring reply for primary instance");
 676 
 677     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 678         if (reply_to != NULL) {
 679             crm_debug("Processing %s from %s", op, originator);
 680             *needs_reply = FALSE;
 681 
 682         } else {
 683             crm_debug("Processing %s reply from %s", op, originator);
 684         }
 685         return TRUE;
 686 
 687     } else {
 688         crm_err("Nothing for us to do?");
 689         crm_log_xml_err(request, "Peer[inbound]");
 690     }
 691 
 692     return FALSE;
 693 }
 694 
 695 static gboolean
 696 parse_peer_options_v2(int call_type, xmlNode * request,
     /*  */
 697                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 698                    gboolean * needs_forward)
 699 {
 700     const char *host = NULL;
 701     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
 702     const char *op = crm_element_value(request, F_CIB_OPERATION);
 703     const char *originator = crm_element_value(request, F_ORIG);
 704     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 705 
 706     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 707 
 708     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
 709         /* sync_our_cib() sets F_CIB_ISREPLY */
 710         if (reply_to) {
 711             delegated = reply_to;
 712         }
 713         goto skip_is_reply;
 714 
 715     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
 716                             pcmk__str_none)) {
 717         // Nothing to do
 718 
 719     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 720         process_ping_reply(request);
 721         return FALSE;
 722 
 723     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
 724         /* Only the DC (node with the oldest software) should process
 725          * this operation if F_CIB_SCHEMA_MAX is unset
 726          *
 727          * If the DC is happy it will then send out another
 728          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
 729          * upgrade.
 730          *
 731          * Except this time F_CIB_SCHEMA_MAX will be set which puts a
 732          * limit on how far newer nodes will go
 733          */
 734         const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
 735         const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
 736 
 737         crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
 738                   op, (is_reply? " reply" : ""),
 739                   (based_is_primary? "primary" : "secondary"),
 740                   (max? max : "none"), (upgrade_rc? upgrade_rc : "none"));
 741 
 742         if (upgrade_rc != NULL) {
 743             // Our upgrade request was rejected by DC, notify clients of result
 744             crm_xml_add(request, F_CIB_RC, upgrade_rc);
 745 
 746         } else if ((max == NULL) && based_is_primary) {
 747             /* We are the DC, check if this upgrade is allowed */
 748             goto skip_is_reply;
 749 
 750         } else if(max) {
 751             /* Ok, go ahead and upgrade to 'max' */
 752             goto skip_is_reply;
 753 
 754         } else {
 755             // Ignore broadcast client requests when we're not DC
 756             return FALSE;
 757         }
 758 
 759     } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 760         crm_info("Detected legacy %s global update from %s", op, originator);
 761         send_sync_request(NULL);
 762         legacy_mode = TRUE;
 763         return FALSE;
 764 
 765     } else if (is_reply && cib_op_modifies(call_type)) {
 766         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 767         return FALSE;
 768 
 769     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 770         /* Legacy handling */
 771         crm_debug("Legacy handling of %s message from %s", op, originator);
 772         *local_notify = FALSE;
 773         if (reply_to == NULL) {
 774             *process = TRUE;
 775         }
 776         return *process;
 777     }
 778 
 779     if(is_reply) {
 780         crm_trace("Handling %s reply sent from %s to local clients", op, originator);
 781         *process = FALSE;
 782         *needs_reply = FALSE;
 783         *local_notify = TRUE;
 784         return TRUE;
 785     }
 786 
 787   skip_is_reply:
 788     *process = TRUE;
 789     *needs_reply = FALSE;
 790 
 791     if(pcmk__str_eq(delegated, cib_our_uname, pcmk__str_casei)) {
 792         *local_notify = TRUE;
 793     } else {
 794         *local_notify = FALSE;
 795     }
 796 
 797     host = crm_element_value(request, F_CIB_HOST);
 798     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 799         crm_trace("Processing %s request sent to us from %s", op, originator);
 800         *needs_reply = TRUE;
 801         return TRUE;
 802 
 803     } else if (host != NULL) {
 804         /* this is for a specific instance and we're not it */
 805         crm_trace("Ignoring %s operation for instance on %s", op, host);
 806         return FALSE;
 807 
 808     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 809         *needs_reply = TRUE;
 810     }
 811 
 812     crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
 813               crm_element_value(request, F_CIB_CLIENTNAME),
 814               crm_element_value(request, F_CIB_CALLID),
 815               originator, (*local_notify)?"(notify)":"");
 816     return TRUE;
 817 }
 818 
 819 static gboolean
 820 parse_peer_options(int call_type, xmlNode * request,
     /*  */
 821                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 822                    gboolean * needs_forward)
 823 {
 824     /* TODO: What happens when an update comes in after node A
 825      * requests the CIB from node B, but before it gets the reply (and
 826      * sends out the replace operation)
 827      */
 828     if(cib_legacy_mode()) {
 829         return parse_peer_options_v1(
 830             call_type, request, local_notify, needs_reply, process, needs_forward);
 831     } else {
 832         return parse_peer_options_v2(
 833             call_type, request, local_notify, needs_reply, process, needs_forward);
 834     }
 835 }
 836 
 837 static void
 838 forward_request(xmlNode *request, int call_options)
     /*  */
 839 {
 840     const char *op = crm_element_value(request, F_CIB_OPERATION);
 841     const char *host = crm_element_value(request, F_CIB_HOST);
 842 
 843     crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);
 844 
 845     if (host != NULL) {
 846         crm_trace("Forwarding %s op to %s", op, host);
 847         send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);
 848 
 849     } else {
 850         crm_trace("Forwarding %s op to primary instance", op);
 851         send_cluster_message(NULL, crm_msg_cib, request, FALSE);
 852     }
 853 
 854     /* Return the request to its original state */
 855     xml_remove_prop(request, F_CIB_DELEGATED);
 856 
 857     if (call_options & cib_discard_reply) {
 858         crm_trace("Client not interested in reply");
 859     }
 860 }
 861 
 862 static gboolean
 863 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
     /*  */
 864 {
 865     CRM_ASSERT(msg != NULL);
 866 
 867     if (broadcast) {
 868         /* this (successful) call modified the CIB _and_ the
 869          * change needs to be broadcast...
 870          *   send via HA to other nodes
 871          */
 872         int diff_add_updates = 0;
 873         int diff_add_epoch = 0;
 874         int diff_add_admin_epoch = 0;
 875 
 876         int diff_del_updates = 0;
 877         int diff_del_epoch = 0;
 878         int diff_del_admin_epoch = 0;
 879 
 880         const char *digest = NULL;
 881         int format = 1;
 882 
 883         CRM_LOG_ASSERT(result_diff != NULL);
 884         digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
 885         crm_element_value_int(result_diff, "format", &format);
 886 
 887         cib_diff_version_details(result_diff,
 888                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
 889                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 890 
 891         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
 892                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
 893                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 894 
 895         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 896         pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
 897         crm_xml_add(msg, F_CIB_OPERATION, PCMK__CIB_REQUEST_APPLY_PATCH);
 898         crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
 899 
 900         if (format == 1) {
 901             CRM_ASSERT(digest != NULL);
 902         }
 903 
 904         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
 905         crm_log_xml_explicit(msg, "copy");
 906         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 907 
 908     } else if (originator != NULL) {
 909         /* send reply via HA to originating node */
 910         crm_trace("Sending request result to %s only", originator);
 911         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 912         return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
 913     }
 914 
 915     return FALSE;
 916 }
 917 
 918 /*!
 919  * \internal
 920  * \brief Handle an IPC or CPG message containing a request
 921  *
 922  * \param[in,out] request        Request XML
 923  * \param[in] privileged         Whether privileged commands may be run
 924  *                               (see cib_server_ops[] definition)
 925  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
 926  */
 927 static void
 928 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
 929                     const pcmk__client_t *cib_client)
 930 {
 931     int call_type = 0;
 932     int call_options = 0;
 933 
 934     gboolean process = TRUE;        // Whether to process request locally now
 935     gboolean is_update = TRUE;      // Whether request would modify CIB
 936     gboolean needs_reply = TRUE;    // Whether to build a reply
 937     gboolean local_notify = FALSE;  // Whether to notify (local) requester
 938     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 939 
 940     xmlNode *op_reply = NULL;
 941     xmlNode *result_diff = NULL;
 942 
 943     int rc = pcmk_ok;
 944     const char *op = crm_element_value(request, F_CIB_OPERATION);
 945     const char *originator = crm_element_value(request, F_ORIG);
 946     const char *host = crm_element_value(request, F_CIB_HOST);
 947     const char *target = NULL;
 948     const char *call_id = crm_element_value(request, F_CIB_CALLID);
 949     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
 950     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
 951     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 952 
 953     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 954 
 955     if ((host != NULL) && (*host == '\0')) {
 956         host = NULL;
 957     }
 958 
 959     if (host) {
 960         target = host;
 961 
 962     } else if (call_options & cib_scope_local) {
 963         target = "local host";
 964 
 965     } else {
 966         target = "primary";
 967     }
 968 
 969     if (cib_client == NULL) {
 970         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
 971                   op, client_name, call_id, originator, target, reply_to);
 972     } else {
 973         crm_xml_add(request, F_ORIG, cib_our_uname);
 974         crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
 975     }
 976 
 977     rc = cib_get_operation_id(op, &call_type);
 978     if (rc != pcmk_ok) {
 979         /* TODO: construct error reply? */
 980         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
 981         return;
 982     }
 983 
 984     if (cib_client != NULL) {
 985         parse_local_options(cib_client, call_type, call_options, host, op,
 986                             &local_notify, &needs_reply, &process, &needs_forward);
 987 
 988     } else if (parse_peer_options(call_type, request, &local_notify,
 989                                   &needs_reply, &process, &needs_forward) == FALSE) {
 990         return;
 991     }
 992 
 993     is_update = cib_op_modifies(call_type);
 994 
 995     if (call_options & cib_discard_reply) {
 996         /* If the request will modify the CIB, and we are in legacy mode, we
 997          * need to build a reply so we can broadcast a diff, even if the
 998          * requester doesn't want one.
 999          */
1000         needs_reply = is_update && cib_legacy_mode();
1001         local_notify = FALSE;
1002     }
1003 
1004     if (needs_forward) {
1005         const char *section = crm_element_value(request, F_CIB_SECTION);
1006         int log_level = LOG_INFO;
1007 
1008         if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
1009             log_level = LOG_DEBUG;
1010         }
1011 
1012         do_crm_log(log_level,
1013                    "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
1014                    op,
1015                    section ? section : "'all'",
1016                    pcmk__s(host, (cib_legacy_mode() ? "primary" : "all")),
1017                    originator ? originator : "local",
1018                    client_name, call_id);
1019 
1020         forward_request(request, call_options);
1021         return;
1022     }
1023 
1024     if (cib_status != pcmk_ok) {
1025         const char *call = crm_element_value(request, F_CIB_CALLID);
1026 
1027         rc = cib_status;
1028         crm_err("Operation ignored, cluster configuration is invalid."
1029                 " Please repair and restart: %s", pcmk_strerror(cib_status));
1030 
1031         op_reply = create_xml_node(NULL, "cib-reply");
1032         crm_xml_add(op_reply, F_TYPE, T_CIB);
1033         crm_xml_add(op_reply, F_CIB_OPERATION, op);
1034         crm_xml_add(op_reply, F_CIB_CALLID, call);
1035         crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
1036         crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
1037         crm_xml_add_int(op_reply, F_CIB_RC, rc);
1038 
1039         crm_trace("Attaching reply output");
1040         add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);
1041 
1042         crm_log_xml_explicit(op_reply, "cib:reply");
1043 
1044     } else if (process) {
1045         time_t finished = 0;
1046         time_t now = time(NULL);
1047         int level = LOG_INFO;
1048         const char *section = crm_element_value(request, F_CIB_SECTION);
1049 
1050         rc = cib_process_command(request, &op_reply, &result_diff, privileged);
1051 
1052         if (!is_update) {
1053             level = LOG_TRACE;
1054 
1055         } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1056             switch (rc) {
1057                 case pcmk_ok:
1058                     level = LOG_INFO;
1059                     break;
1060                 case -pcmk_err_old_data:
1061                 case -pcmk_err_diff_resync:
1062                 case -pcmk_err_diff_failed:
1063                     level = LOG_TRACE;
1064                     break;
1065                 default:
1066                     level = LOG_ERR;
1067             }
1068 
1069         } else if (rc != pcmk_ok) {
1070             level = LOG_WARNING;
1071         }
1072 
1073         do_crm_log(level,
1074                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
1075                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
1076                    originator ? originator : "local", client_name, call_id,
1077                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
1078                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
1079                    the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
1080 
1081         finished = time(NULL);
1082         if ((finished - now) > 3) {
1083             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
1084             crm_write_blackbox(0, NULL);
1085         }
1086 
1087         if (op_reply == NULL && (needs_reply || local_notify)) {
1088             crm_err("Unexpected NULL reply to message");
1089             crm_log_xml_err(request, "null reply");
1090             needs_reply = FALSE;
1091             local_notify = FALSE;
1092         }
1093     }
1094 
1095     if (is_update && !cib_legacy_mode()) {
1096         crm_trace("Completed pre-sync update from %s/%s/%s%s",
1097                   originator ? originator : "local", client_name, call_id,
1098                   local_notify?" with local notification":"");
1099 
1100     } else if (!needs_reply || stand_alone) {
1101         // This was a non-originating secondary update
1102         crm_trace("Completed update as secondary");
1103 
1104     } else if (cib_legacy_mode() &&
1105                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
1106         gboolean broadcast = FALSE;
1107 
1108         cib_local_bcast_num++;
1109         crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
1110         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
1111 
1112         if (broadcast && client_id && local_notify && op_reply) {
1113 
1114             /* If we have been asked to sync the reply,
1115              * and a bcast msg has gone out, we queue the local notify
1116              * until we know the bcast message has been received */
1117             local_notify = FALSE;
1118             crm_trace("Queuing local %ssync notification for %s",
1119                       (call_options & cib_sync_call) ? "" : "a-", client_id);
1120 
1121             queue_local_notify(op_reply, client_id,
1122                                pcmk_is_set(call_options, cib_sync_call),
1123                                (cib_client == NULL));
1124             op_reply = NULL;    /* the reply is queued, so don't free here */
1125         }
1126 
1127     } else if (call_options & cib_discard_reply) {
1128         crm_trace("Caller isn't interested in reply");
1129 
1130     } else if (cib_client == NULL) {
1131         if (is_update == FALSE || result_diff == NULL) {
1132             crm_trace("Request not broadcast: R/O call");
1133 
1134         } else if (call_options & cib_inhibit_bcast) {
1135             crm_trace("Request not broadcast: inhibited");
1136 
1137         } else if (rc != pcmk_ok) {
1138             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
1139 
1140         } else {
1141             crm_trace("Directing reply to %s", originator);
1142         }
1143 
1144         send_peer_reply(op_reply, result_diff, originator, FALSE);
1145     }
1146 
1147     if (local_notify && client_id) {
1148         crm_trace("Performing local %ssync notification for %s",
1149                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
1150                   client_id);
1151         if (process == FALSE) {
1152             do_local_notify(request, client_id,
1153                             pcmk_is_set(call_options, cib_sync_call),
1154                             (cib_client == NULL));
1155         } else {
1156             do_local_notify(op_reply, client_id,
1157                             pcmk_is_set(call_options, cib_sync_call),
1158                             (cib_client == NULL));
1159         }
1160     }
1161 
1162     free_xml(op_reply);
1163     free_xml(result_diff);
1164 
1165     return;
1166 }
1167 
1168 static char *
1169 calculate_section_digest(const char *xpath, xmlNode * xml_obj)
     /*  */
1170 {
1171     xmlNode *xml_section = NULL;
1172 
1173     if (xml_obj == NULL) {
1174         return NULL;
1175     }
1176 
1177     xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
1178     if (xml_section == NULL) {
1179         return NULL;
1180     }
1181     return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET); 
1182 
1183 }
1184 
1185 static int
1186 cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gboolean privileged)
     /*  */
1187 {
1188     xmlNode *input = NULL;
1189     xmlNode *output = NULL;
1190     xmlNode *result_cib = NULL;
1191     xmlNode *current_cib = NULL;
1192 
1193     int call_type = 0;
1194     int call_options = 0;
1195 
1196     const char *op = NULL;
1197     const char *section = NULL;
1198     const char *call_id = crm_element_value(request, F_CIB_CALLID);
1199 
1200     int rc = pcmk_ok;
1201     int rc2 = pcmk_ok;
1202 
1203     gboolean send_r_notify = FALSE;
1204     gboolean global_update = FALSE;
1205     gboolean config_changed = FALSE;
1206     gboolean manage_counters = TRUE;
1207 
1208     static mainloop_timer_t *digest_timer = NULL;
1209 
1210     char *current_nodes_digest = NULL;
1211     char *current_alerts_digest = NULL;
1212     char *current_status_digest = NULL;
1213     uint32_t change_section = cib_change_section_nodes
1214                               |cib_change_section_alerts
1215                               |cib_change_section_status;
1216 
1217     CRM_ASSERT(cib_status == pcmk_ok);
1218 
1219     if(digest_timer == NULL) {
1220         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1221     }
1222 
1223     *reply = NULL;
1224     *cib_diff = NULL;
1225     current_cib = the_cib;
1226 
1227     /* Start processing the request... */
1228     op = crm_element_value(request, F_CIB_OPERATION);
1229     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
1230     rc = cib_get_operation_id(op, &call_type);
1231 
1232     if (rc == pcmk_ok && privileged == FALSE) {
1233         rc = cib_op_can_run(call_type, call_options, privileged, global_update);
1234     }
1235 
1236     rc2 = cib_op_prepare(call_type, request, &input, &section);
1237     if (rc == pcmk_ok) {
1238         rc = rc2;
1239     }
1240 
1241     if (rc != pcmk_ok) {
1242         crm_trace("Call setup failed: %s", pcmk_strerror(rc));
1243         goto done;
1244 
1245     } else if (cib_op_modifies(call_type) == FALSE) {
1246         rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
1247                             section, request, input, FALSE, &config_changed,
1248                             current_cib, &result_cib, NULL, &output);
1249 
1250         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
1251         goto done;
1252     }
1253 
1254     /* Handle a valid write action */
1255     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1256         /* legacy code */
1257         manage_counters = FALSE;
1258         cib__set_call_options(call_options, "call", cib_force_diff);
1259         crm_trace("Global update detected");
1260 
1261         CRM_CHECK(call_type == 3 || call_type == 4, crm_err("Call type: %d", call_type);
1262                   crm_log_xml_err(request, "bad op"));
1263     }
1264 
1265     if (rc == pcmk_ok) {
1266         ping_modified_since = TRUE;
1267         if (call_options & cib_inhibit_bcast) {
1268             /* skip */
1269             crm_trace("Skipping update: inhibit broadcast");
1270             manage_counters = FALSE;
1271         }
1272 
1273         if (!pcmk_is_set(call_options, cib_dryrun)
1274             && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
1275             /* Copying large CIBs accounts for a huge percentage of our CIB usage */
1276             cib__set_call_options(call_options, "call", cib_zero_copy);
1277         } else {
1278             cib__clear_call_options(call_options, "call", cib_zero_copy);
1279         }
1280 
1281         /* Calculate the hash value of the section before the change. */
1282         if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
1283             current_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, current_cib);
1284             current_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, current_cib);
1285             current_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, current_cib);
1286             crm_trace("current-digest %s:%s:%s", current_nodes_digest, current_alerts_digest, current_status_digest);
1287         }
1288 
1289         /* result_cib must not be modified after cib_perform_op() returns */
1290         rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
1291                             section, request, input, manage_counters, &config_changed,
1292                             current_cib, &result_cib, cib_diff, &output);
1293 
1294         if (manage_counters == FALSE) {
1295             int format = 1;
1296             /* Legacy code
1297              * If the diff is NULL at this point, it's because nothing changed
1298              */
1299             if (*cib_diff) {
1300                 crm_element_value_int(*cib_diff, "format", &format);
1301             }
1302 
1303             if (format == 1) {
1304                 config_changed = cib_config_changed(NULL, NULL, cib_diff);
1305             }
1306         }
1307 
1308         /* Always write to disk for replace ops,
1309          * this also negates the need to detect ordering changes
1310          */
1311         if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
1312             config_changed = TRUE;
1313         }
1314     }
1315 
1316     if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
1317         crm_trace("Activating %s->%s%s%s",
1318                   crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
1319                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
1320                   (pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
1321                   (config_changed? " changed" : ""));
1322         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1323             rc = activateCibXml(result_cib, config_changed, op);
1324             crm_trace("Activated %s (%d)",
1325                       crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
1326         }
1327 
1328         if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
1329             cib_read_config(config_hash, result_cib);
1330         }
1331 
1332         if (pcmk__str_eq(PCMK__CIB_REQUEST_REPLACE, op, pcmk__str_none)) {
1333             char *result_nodes_digest = NULL;
1334             char *result_alerts_digest = NULL;
1335             char *result_status_digest = NULL;
1336 
1337             /* Calculate the hash value of the changed section. */
1338             result_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, result_cib);
1339             result_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, result_cib);
1340             result_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, result_cib);
1341             crm_trace("result-digest %s:%s:%s", result_nodes_digest, result_alerts_digest, result_status_digest);
1342 
1343             if (pcmk__str_eq(current_nodes_digest, result_nodes_digest,
1344                              pcmk__str_none)) {
1345                 change_section =
1346                     pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
1347                                          "CIB change section",
1348                                          "change_section", change_section,
1349                                          cib_change_section_nodes, "nodes");
1350             }
1351 
1352             if (pcmk__str_eq(current_alerts_digest, result_alerts_digest,
1353                              pcmk__str_none)) {
1354                 change_section =
1355                     pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
1356                                          "CIB change section",
1357                                          "change_section", change_section,
1358                                          cib_change_section_alerts, "alerts");
1359             }
1360 
1361             if (pcmk__str_eq(current_status_digest, result_status_digest,
1362                              pcmk__str_none)) {
1363                 change_section =
1364                     pcmk__clear_flags_as(__func__, __LINE__, LOG_TRACE,
1365                                          "CIB change section",
1366                                          "change_section", change_section,
1367                                          cib_change_section_status, "status");
1368             }
1369 
1370             if (change_section != cib_change_section_none) {
1371                 send_r_notify = TRUE;
1372             }
1373             
1374             free(result_nodes_digest);
1375             free(result_alerts_digest);
1376             free(result_status_digest);
1377 
1378         } else if (pcmk__str_eq(PCMK__CIB_REQUEST_ERASE, op, pcmk__str_none)) {
1379             send_r_notify = TRUE;
1380         }
1381 
1382         mainloop_timer_stop(digest_timer);
1383         mainloop_timer_start(digest_timer);
1384 
1385     } else if (rc == -pcmk_err_schema_validation) {
1386         CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));
1387 
1388         if (output != NULL) {
1389             crm_log_xml_info(output, "cib:output");
1390             free_xml(output);
1391         }
1392 
1393         output = result_cib;
1394 
1395     } else {
1396         crm_trace("Not activating %d %d %s", rc,
1397                   pcmk_is_set(call_options, cib_dryrun),
1398                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
1399         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1400             free_xml(result_cib);
1401         }
1402     }
1403 
1404     if ((call_options & (cib_inhibit_notify|cib_dryrun)) == 0) {
1405         const char *client = crm_element_value(request, F_CIB_CLIENTNAME);
1406 
1407         crm_trace("Sending notifications %d",
1408                   pcmk_is_set(call_options, cib_dryrun));
1409         cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
1410     }
1411 
1412     if (send_r_notify) {
1413         const char *origin = crm_element_value(request, F_ORIG);
1414 
1415         cib_replace_notify(origin, the_cib, rc, *cib_diff, change_section);
1416     }
1417 
1418     xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
1419   done:
1420     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
1421         const char *caller = crm_element_value(request, F_CIB_CLIENTID);
1422 
1423         *reply = create_xml_node(NULL, "cib-reply");
1424         crm_xml_add(*reply, F_TYPE, T_CIB);
1425         crm_xml_add(*reply, F_CIB_OPERATION, op);
1426         crm_xml_add(*reply, F_CIB_CALLID, call_id);
1427         crm_xml_add(*reply, F_CIB_CLIENTID, caller);
1428         crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
1429         crm_xml_add_int(*reply, F_CIB_RC, rc);
1430 
1431         if (output != NULL) {
1432             crm_trace("Attaching reply output");
1433             add_message_xml(*reply, F_CIB_CALLDATA, output);
1434         }
1435 
1436         crm_log_xml_explicit(*reply, "cib:reply");
1437     }
1438 
1439     crm_trace("cleanup");
1440 
1441     if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
1442         free_xml(output);
1443         output = NULL;
1444     }
1445 
1446     if (call_type >= 0) {
1447         cib_op_cleanup(call_type, call_options, &input, &output);
1448     }
1449 
1450     free(current_nodes_digest);
1451     free(current_alerts_digest);
1452     free(current_status_digest);
1453 
1454     crm_trace("done");
1455     return rc;
1456 }
1457 
1458 void
1459 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1460 {
1461     const char *reason = NULL;
1462     const char *originator = crm_element_value(msg, F_ORIG);
1463 
1464     if (cib_legacy_mode() && pcmk__str_eq(originator, cib_our_uname, pcmk__str_null_matches)) {
1465         /* message is from ourselves */
1466         int bcast_id = 0;
1467 
1468         if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
1469             check_local_notify(bcast_id);
1470         }
1471         return;
1472 
1473     } else if (crm_peer_cache == NULL) {
1474         reason = "membership not established";
1475         goto bail;
1476     }
1477 
1478     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
1479         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
1480     }
1481 
1482     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1483     cib_process_request(msg, TRUE, NULL);
1484     return;
1485 
1486   bail:
1487     if (reason) {
1488         const char *seq = crm_element_value(msg, F_SEQ);
1489         const char *op = crm_element_value(msg, F_CIB_OPERATION);
1490 
1491         crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
1492     }
1493 }
1494 
1495 static gboolean
1496 cib_force_exit(gpointer data)
     /*  */
1497 {
1498     crm_notice("Forcing exit!");
1499     terminate_cib(__func__, CRM_EX_ERROR);
1500     return FALSE;
1501 }
1502 
1503 static void
1504 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1505 {
1506     pcmk__client_t *a_client = value;
1507 
1508     crm_err("Can't disconnect client %s: Not implemented",
1509             pcmk__client_name(a_client));
1510 }
1511 
1512 void
1513 cib_shutdown(int nsig)
     /*  */
1514 {
1515     struct qb_ipcs_stats srv_stats;
1516 
1517     if (cib_shutdown_flag == FALSE) {
1518         int disconnects = 0;
1519         qb_ipcs_connection_t *c = NULL;
1520 
1521         cib_shutdown_flag = TRUE;
1522 
1523         c = qb_ipcs_connection_first_get(ipcs_rw);
1524         while (c != NULL) {
1525             qb_ipcs_connection_t *last = c;
1526 
1527             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1528 
1529             crm_debug("Disconnecting r/w client %p...", last);
1530             qb_ipcs_disconnect(last);
1531             qb_ipcs_connection_unref(last);
1532             disconnects++;
1533         }
1534 
1535         c = qb_ipcs_connection_first_get(ipcs_ro);
1536         while (c != NULL) {
1537             qb_ipcs_connection_t *last = c;
1538 
1539             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1540 
1541             crm_debug("Disconnecting r/o client %p...", last);
1542             qb_ipcs_disconnect(last);
1543             qb_ipcs_connection_unref(last);
1544             disconnects++;
1545         }
1546 
1547         c = qb_ipcs_connection_first_get(ipcs_shm);
1548         while (c != NULL) {
1549             qb_ipcs_connection_t *last = c;
1550 
1551             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1552 
1553             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1554             qb_ipcs_disconnect(last);
1555             qb_ipcs_connection_unref(last);
1556             disconnects++;
1557         }
1558 
1559         disconnects += pcmk__ipc_client_count();
1560 
1561         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1562         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1563         crm_info("Disconnected %d clients", disconnects);
1564     }
1565 
1566     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1567 
1568     if (pcmk__ipc_client_count() == 0) {
1569         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1570         initiate_exit();
1571 
1572     } else {
1573         crm_info("Waiting on %d clients to disconnect (%d)",
1574                  pcmk__ipc_client_count(), srv_stats.active_connections);
1575     }
1576 }
1577 
1578 void
1579 initiate_exit(void)
     /*  */
1580 {
1581     int active = 0;
1582     xmlNode *leaving = NULL;
1583 
1584     active = crm_active_peers();
1585     if (active < 2) {
1586         terminate_cib(__func__, 0);
1587         return;
1588     }
1589 
1590     crm_info("Sending disconnect notification to %d peers...", active);
1591 
1592     leaving = create_xml_node(NULL, "exit-notification");
1593     crm_xml_add(leaving, F_TYPE, "cib");
1594     crm_xml_add(leaving, F_CIB_OPERATION, PCMK__CIB_REQUEST_SHUTDOWN);
1595 
1596     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
1597     free_xml(leaving);
1598 
1599     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1600 }
1601 
1602 extern int remote_fd;
1603 extern int remote_tls_fd;
1604 
1605 /*!
1606  * \internal
1607  * \brief Close remote sockets, free the global CIB and quit
1608  *
1609  * \param[in] caller           Name of calling function (for log message)
1610  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1611  */
1612 void
1613 terminate_cib(const char *caller, int fast)
     /*  */
1614 {
1615     crm_info("%s: Exiting%s...", caller,
1616              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1617 
1618     if (remote_fd > 0) {
1619         close(remote_fd);
1620         remote_fd = 0;
1621     }
1622     if (remote_tls_fd > 0) {
1623         close(remote_tls_fd);
1624         remote_tls_fd = 0;
1625     }
1626 
1627     uninitializeCib();
1628 
1629     if (fast > 0) {
1630         /* Quit fast on error */
1631         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1632         crm_exit(fast);
1633 
1634     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1635         /* Quit via returning from the main loop. If fast == -1, we skip the
1636          * disconnect here, and it will be done when the main loop returns
1637          * (this allows the peer status callback to avoid messing with the
1638          * peer caches).
1639          */
1640         if (fast == 0) {
1641             crm_cluster_disconnect(&crm_cluster);
1642         }
1643         g_main_loop_quit(mainloop);
1644 
1645     } else {
1646         /* Quit via clean exit. Even the peer status callback can disconnect
1647          * here, because we're not returning control to the caller. */
1648         crm_cluster_disconnect(&crm_cluster);
1649         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1650         crm_exit(CRM_EX_OK);
1651     }
1652 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
