daemons/based/based

/* */
This source file includes following definitions.
cib_legacy_mode
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
do_local_notify
local_notify_destroy_callback
check_local_notify
queue_local_notify
parse_local_options_v1
parse_local_options_v2
parse_local_options
parse_peer_options_v1
parse_peer_options_v2
parse_peer_options
forward_request
send_peer_reply
cib_process_request
calculate_section_digest
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
cib_shutdown
initiate_exit
terminate_cib
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>  /* U64T ~ PRIu64 */
  22 
  23 #include <crm/crm.h>
  24 #include <crm/cib.h>
  25 #include <crm/msg_xml.h>
  26 #include <crm/cluster/internal.h>
  27 
  28 #include <crm/common/xml.h>
  29 #include <crm/common/remote_internal.h>
  30 
  31 #include <pacemaker-based.h>
  32 
  33 #define EXIT_ESCALATION_MS 10000
  34 
  35 static unsigned long cib_local_bcast_num = 0;
  36 
  37 typedef struct cib_local_notify_s {
  38     xmlNode *notify_src;
  39     char *client_id;
  40     gboolean from_peer;
  41     gboolean sync_reply;
  42 } cib_local_notify_t;
  43 
  44 int next_client_id = 0;
  45 
  46 gboolean legacy_mode = FALSE;
  47 
  48 qb_ipcs_service_t *ipcs_ro = NULL;
  49 qb_ipcs_service_t *ipcs_rw = NULL;
  50 qb_ipcs_service_t *ipcs_shm = NULL;
  51 
  52 void send_cib_replace(const xmlNode * sync_request, const char *host);
  53 static void cib_process_request(xmlNode *request, gboolean privileged,
  54                                 pcmk__client_t *cib_client);
  55 
  56 
  57 static int cib_process_command(xmlNode *request, xmlNode **reply,
  58                                xmlNode **cib_diff, gboolean privileged);
  59 
  60 gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
  61                              gboolean privileged);
  62 
  63 gboolean cib_legacy_mode(void)
     /*  */
  64 {
  65     return legacy_mode;
  66 }
  67 
  68 
  69 static int32_t
  70 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  71 {
  72     if (cib_shutdown_flag) {
  73         crm_info("Ignoring new IPC client [%d] during shutdown",
  74                  pcmk__client_pid(c));
  75         return -EPERM;
  76     }
  77 
  78     if (pcmk__new_client(c, uid, gid) == NULL) {
  79         return -EIO;
  80     }
  81     return 0;
  82 }
  83 
  84 static int32_t
  85 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  86 {
  87     pcmk__client_t *client = pcmk__find_client(c);
  88 
  89     crm_trace("%p message from %s", c, client->id);
  90     return cib_common_callback(c, data, size, TRUE);
  91 }
  92 
  93 static int32_t
  94 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  95 {
  96     pcmk__client_t *client = pcmk__find_client(c);
  97 
  98     crm_trace("%p message from %s", c, client->id);
  99     return cib_common_callback(c, data, size, FALSE);
 100 }
 101 
 102 /* Error code means? */
 103 static int32_t
 104 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 105 {
 106     pcmk__client_t *client = pcmk__find_client(c);
 107 
 108     if (client == NULL) {
 109         return 0;
 110     }
 111     crm_trace("Connection %p", c);
 112     pcmk__free_client(client);
 113     return 0;
 114 }
 115 
 116 static void
 117 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 118 {
 119     crm_trace("Connection %p", c);
 120     cib_ipc_closed(c);
 121     if (cib_shutdown_flag) {
 122         cib_shutdown(0);
 123     }
 124 }
 125 
 126 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 127     .connection_accept = cib_ipc_accept,
 128     .connection_created = NULL,
 129     .msg_process = cib_ipc_dispatch_ro,
 130     .connection_closed = cib_ipc_closed,
 131     .connection_destroyed = cib_ipc_destroy
 132 };
 133 
 134 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 135     .connection_accept = cib_ipc_accept,
 136     .connection_created = NULL,
 137     .msg_process = cib_ipc_dispatch_rw,
 138     .connection_closed = cib_ipc_closed,
 139     .connection_destroyed = cib_ipc_destroy
 140 };
 141 
 142 void
 143 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 144                            pcmk__client_t *cib_client, gboolean privileged)
 145 {
 146     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 147 
 148     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 149         if (flags & crm_ipc_client_response) {
 150             xmlNode *ack = create_xml_node(NULL, __func__);
 151 
 152             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
 153             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
 154             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 155             cib_client->request_id = 0;
 156             free_xml(ack);
 157         }
 158         return;
 159 
 160     } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
 161         /* Update the notify filters for this client */
 162         int on_off = 0;
 163         crm_exit_t status = CRM_EX_OK;
 164         uint64_t bit = UINT64_C(0);
 165         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 166 
 167         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 168 
 169         crm_debug("Setting %s callbacks %s for client %s",
 170                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 171 
 172         if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
 173             bit = cib_notify_post;
 174 
 175         } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
 176             bit = cib_notify_pre;
 177 
 178         } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
 179             bit = cib_notify_confirm;
 180 
 181         } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
 182             bit = cib_notify_diff;
 183 
 184         } else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
 185             bit = cib_notify_replace;
 186 
 187         } else {
 188             status = CRM_EX_INVALID_PARAM;
 189         }
 190 
 191         if (bit != 0) {
 192             if (on_off) {
 193                 pcmk__set_client_flags(cib_client, bit);
 194             } else {
 195                 pcmk__clear_client_flags(cib_client, bit);
 196             }
 197         }
 198 
 199         pcmk__ipc_send_ack(cib_client, id, flags, "ack", status);
 200         return;
 201     }
 202 
 203     cib_process_request(op_request, privileged, cib_client);
 204 }
 205 
 206 int32_t
 207 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 208 {
 209     uint32_t id = 0;
 210     uint32_t flags = 0;
 211     int call_options = 0;
 212     pcmk__client_t *cib_client = pcmk__find_client(c);
 213     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 214 
 215     if (op_request) {
 216         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 217     }
 218 
 219     if (op_request == NULL) {
 220         crm_trace("Invalid message from %p", c);
 221         pcmk__ipc_send_ack(cib_client, id, flags, "nack", CRM_EX_PROTOCOL);
 222         return 0;
 223 
 224     } else if(cib_client == NULL) {
 225         crm_trace("Invalid client %p", c);
 226         return 0;
 227     }
 228 
 229     if (pcmk_is_set(call_options, cib_sync_call)) {
 230         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 231         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 232         cib_client->request_id = id;    /* Reply only to the last one */
 233     }
 234 
 235     if (cib_client->name == NULL) {
 236         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 237 
 238         if (value == NULL) {
 239             cib_client->name = pcmk__itoa(cib_client->pid);
 240         } else {
 241             cib_client->name = strdup(value);
 242             if (crm_is_daemon_name(value)) {
 243                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 244             }
 245         }
 246     }
 247 
 248     /* Allow cluster daemons more leeway before being evicted */
 249     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 250         const char *qmax = cib_config_lookup("cluster-ipc-limit");
 251 
 252         if (pcmk__set_client_queue_max(cib_client, qmax)) {
 253             crm_trace("IPC threshold for client %s[%u] is now %u",
 254                       pcmk__client_name(cib_client), cib_client->pid,
 255                       cib_client->queue_max);
 256         }
 257     }
 258 
 259     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
 260     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 261 
 262     CRM_LOG_ASSERT(cib_client->user != NULL);
 263     pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
 264 
 265     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 266     free_xml(op_request);
 267 
 268     return 0;
 269 }
 270 
 271 static uint64_t ping_seq = 0;
 272 static char *ping_digest = NULL;
 273 static bool ping_modified_since = FALSE;
 274 int sync_our_cib(xmlNode * request, gboolean all);
 275 
 276 static gboolean
 277 cib_digester_cb(gpointer data)
     /*  */
 278 {
 279     if (cib_is_master) {
 280         char buffer[32];
 281         xmlNode *ping = create_xml_node(NULL, "ping");
 282 
 283         ping_seq++;
 284         free(ping_digest);
 285         ping_digest = NULL;
 286         ping_modified_since = FALSE;
 287         snprintf(buffer, 32, "%" U64T, ping_seq);
 288         crm_trace("Requesting peer digests (%s)", buffer);
 289 
 290         crm_xml_add(ping, F_TYPE, "cib");
 291         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
 292         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 293 
 294         crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 295         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 296 
 297         free_xml(ping);
 298     }
 299     return FALSE;
 300 }
 301 
 302 static void
 303 process_ping_reply(xmlNode *reply) 
     /*  */
 304 {
 305     uint64_t seq = 0;
 306     const char *host = crm_element_value(reply, F_ORIG);
 307 
 308     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
 309     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
 310     const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
 311 
 312     if (seq_s == NULL) {
 313         crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
 314         return;
 315 
 316     } else {
 317         long long seq_ll;
 318 
 319         if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
 320             return;
 321         }
 322         seq = (uint64_t) seq_ll;
 323     }
 324 
 325     if(digest == NULL) {
 326         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 327 
 328     } else if(seq != ping_seq) {
 329         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 330 
 331     } else if(ping_modified_since) {
 332         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 333 
 334     } else {
 335         const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
 336 
 337         if(ping_digest == NULL) {
 338             crm_trace("Calculating new digest");
 339             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
 340         }
 341 
 342         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 343         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 344             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
 345 
 346             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 347                        crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
 348                        crm_element_value(the_cib, XML_ATTR_GENERATION),
 349                        crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
 350                        ping_digest, host,
 351                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
 352                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
 353                        remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
 354                        digest, remote_cib);
 355 
 356             if(remote_cib && remote_cib->children) {
 357                 /* Additional debug */
 358                 xml_calculate_changes(the_cib, remote_cib);
 359                 xml_log_changes(LOG_INFO, __func__, remote_cib);
 360                 crm_trace("End of differences");
 361             }
 362 
 363             free_xml(remote_cib);
 364             sync_our_cib(reply, FALSE);
 365         }
 366     }
 367 }
 368 
 369 static void
 370 do_local_notify(xmlNode * notify_src, const char *client_id,
     /*  */
 371                 gboolean sync_reply, gboolean from_peer)
 372 {
 373     int rid = 0;
 374     int call_id = 0;
 375     pcmk__client_t *client_obj = NULL;
 376 
 377     CRM_ASSERT(notify_src && client_id);
 378 
 379     crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
 380 
 381     client_obj = pcmk__find_client_by_id(client_id);
 382     if (client_obj == NULL) {
 383         crm_debug("Could not send response %d: client %s not found",
 384                   call_id, client_id);
 385         return;
 386     }
 387 
 388     if (sync_reply) {
 389         if (client_obj->ipcs) {
 390             CRM_LOG_ASSERT(client_obj->request_id);
 391 
 392             rid = client_obj->request_id;
 393             client_obj->request_id = 0;
 394 
 395             crm_trace("Sending response %d to client %s%s",
 396                       rid, pcmk__client_name(client_obj),
 397                       (from_peer? " (originator of delegated request)" : ""));
 398         } else {
 399             crm_trace("Sending response (call %d) to client %s%s",
 400                       call_id, pcmk__client_name(client_obj),
 401                       (from_peer? " (originator of delegated request)" : ""));
 402         }
 403 
 404     } else {
 405         crm_trace("Sending event %d to client %s%s",
 406                   call_id, pcmk__client_name(client_obj),
 407                   (from_peer? " (originator of delegated request)" : ""));
 408     }
 409 
 410     switch (PCMK__CLIENT_TYPE(client_obj)) {
 411         case pcmk__client_ipc:
 412             {
 413                 int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
 414                                             (sync_reply? crm_ipc_flags_none
 415                                              : crm_ipc_server_event));
 416 
 417                 if (rc != pcmk_rc_ok) {
 418                     crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
 419                              (sync_reply? "Synchronous" : "Asynchronous"),
 420                              pcmk__client_name(client_obj), pcmk_rc_str(rc),
 421                              rc);
 422                 }
 423             }
 424             break;
 425 #ifdef HAVE_GNUTLS_GNUTLS_H
 426         case pcmk__client_tls:
 427 #endif
 428         case pcmk__client_tcp:
 429             pcmk__remote_send_xml(client_obj->remote, notify_src);
 430             break;
 431         default:
 432             crm_err("Unknown transport for client %s "
 433                     CRM_XS " flags=%#016" PRIx64,
 434                     pcmk__client_name(client_obj), client_obj->flags);
 435     }
 436 }
 437 
 438 static void
 439 local_notify_destroy_callback(gpointer data)
     /*  */
 440 {
 441     cib_local_notify_t *notify = data;
 442 
 443     free_xml(notify->notify_src);
 444     free(notify->client_id);
 445     free(notify);
 446 }
 447 
 448 static void
 449 check_local_notify(int bcast_id)
     /*  */
 450 {
 451     cib_local_notify_t *notify = NULL;
 452 
 453     if (!local_notify_queue) {
 454         return;
 455     }
 456 
 457     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 458 
 459     if (notify) {
 460         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
 461                         notify->from_peer);
 462         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
 463     }
 464 }
 465 
 466 static void
 467 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
     /*  */
 468                    gboolean from_peer)
 469 {
 470     cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
 471 
 472     notify->notify_src = notify_src;
 473     notify->client_id = strdup(client_id);
 474     notify->sync_reply = sync_reply;
 475     notify->from_peer = from_peer;
 476 
 477     if (!local_notify_queue) {
 478         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
 479     }
 480     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
 481     // cppcheck doesn't know notify will get freed when hash table is destroyed
 482     // cppcheck-suppress memleak
 483 }
 484 
 485 static void
 486 parse_local_options_v1(pcmk__client_t *cib_client, int call_type,
     /*  */
 487                        int call_options, const char *host, const char *op,
 488                        gboolean *local_notify, gboolean *needs_reply,
 489                        gboolean *process, gboolean *needs_forward)
 490 {
 491     if (cib_op_modifies(call_type)
 492         && !(call_options & cib_inhibit_bcast)) {
 493         /* we need to send an update anyway */
 494         *needs_reply = TRUE;
 495     } else {
 496         *needs_reply = FALSE;
 497     }
 498 
 499     if (host == NULL && (call_options & cib_scope_local)) {
 500         crm_trace("Processing locally scoped %s op from client %s",
 501                   op, pcmk__client_name(cib_client));
 502         *local_notify = TRUE;
 503 
 504     } else if (host == NULL && cib_is_master) {
 505         crm_trace("Processing master %s op locally from client %s",
 506                   op, pcmk__client_name(cib_client));
 507         *local_notify = TRUE;
 508 
 509     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 510         crm_trace("Processing locally addressed %s op from client %s",
 511                   op, pcmk__client_name(cib_client));
 512         *local_notify = TRUE;
 513 
 514     } else if (stand_alone) {
 515         *needs_forward = FALSE;
 516         *local_notify = TRUE;
 517         *process = TRUE;
 518 
 519     } else {
 520         crm_trace("%s op from %s needs to be forwarded to client %s",
 521                   op, pcmk__client_name(cib_client),
 522                   (host? host : "the master instance"));
 523         *needs_forward = TRUE;
 524         *process = FALSE;
 525     }
 526 }
 527 
 528 static void
 529 parse_local_options_v2(pcmk__client_t *cib_client, int call_type,
     /*  */
 530                        int call_options, const char *host, const char *op,
 531                        gboolean *local_notify, gboolean *needs_reply,
 532                        gboolean *process, gboolean *needs_forward)
 533 {
 534     if (cib_op_modifies(call_type)) {
 535         if (pcmk__strcase_any_of(op, CIB_OP_MASTER, CIB_OP_SLAVE, NULL)) {
 536             /* Always handle these locally */
 537             *process = TRUE;
 538             *needs_reply = FALSE;
 539             *local_notify = TRUE;
 540             *needs_forward = FALSE;
 541             return;
 542 
 543         } else {
 544             /* Redirect all other updates via CPG */
 545             *needs_reply = TRUE;
 546             *needs_forward = TRUE;
 547             *process = FALSE;
 548             crm_trace("%s op from %s needs to be forwarded to client %s",
 549                       op, pcmk__client_name(cib_client),
 550                       (host? host : "the master instance"));
 551             return;
 552         }
 553     }
 554 
 555 
 556     *process = TRUE;
 557     *needs_reply = FALSE;
 558     *local_notify = TRUE;
 559     *needs_forward = FALSE;
 560 
 561     if (stand_alone) {
 562         crm_trace("Processing %s op from client %s (stand-alone)",
 563                   op, pcmk__client_name(cib_client));
 564 
 565     } else if (host == NULL) {
 566         crm_trace("Processing unaddressed %s op from client %s",
 567                   op, pcmk__client_name(cib_client));
 568 
 569     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 570         crm_trace("Processing locally addressed %s op from client %s",
 571                   op, pcmk__client_name(cib_client));
 572 
 573     } else {
 574         crm_trace("%s op from %s needs to be forwarded to client %s",
 575                   op, pcmk__client_name(cib_client), host);
 576         *needs_forward = TRUE;
 577         *process = FALSE;
 578     }
 579 }
 580 
 581 static void
 582 parse_local_options(pcmk__client_t *cib_client, int call_type,
     /*  */
 583                     int call_options, const char *host, const char *op,
 584                     gboolean *local_notify, gboolean *needs_reply,
 585                     gboolean *process, gboolean *needs_forward)
 586 {
 587     if(cib_legacy_mode()) {
 588         parse_local_options_v1(cib_client, call_type, call_options, host,
 589                                op, local_notify, needs_reply, process, needs_forward);
 590     } else {
 591         parse_local_options_v2(cib_client, call_type, call_options, host,
 592                                op, local_notify, needs_reply, process, needs_forward);
 593     }
 594 }
 595 
 596 static gboolean
 597 parse_peer_options_v1(int call_type, xmlNode * request,
     /*  */
 598                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 599                    gboolean * needs_forward)
 600 {
 601     const char *op = NULL;
 602     const char *host = NULL;
 603     const char *delegated = NULL;
 604     const char *originator = crm_element_value(request, F_ORIG);
 605     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 606 
 607     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 608 
 609     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 610         *needs_reply = FALSE;
 611         if (is_reply) {
 612             *local_notify = TRUE;
 613             crm_trace("Processing global/peer update from %s"
 614                       " that originated from us", originator);
 615         } else {
 616             crm_trace("Processing global/peer update from %s", originator);
 617         }
 618         return TRUE;
 619     }
 620 
 621     op = crm_element_value(request, F_CIB_OPERATION);
 622     crm_trace("Processing %s request sent by %s", op, originator);
 623     if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 624         /* Always process these */
 625         *local_notify = FALSE;
 626         if (reply_to == NULL || is_reply) {
 627             *process = TRUE;
 628         }
 629         if (is_reply) {
 630             *needs_reply = FALSE;
 631         }
 632         return *process;
 633     }
 634 
 635     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 636         process_ping_reply(request);
 637         return FALSE;
 638     }
 639 
 640     if (is_reply) {
 641         crm_trace("Forward reply sent from %s to local clients", originator);
 642         *process = FALSE;
 643         *needs_reply = FALSE;
 644         *local_notify = TRUE;
 645         return TRUE;
 646     }
 647 
 648     host = crm_element_value(request, F_CIB_HOST);
 649     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 650         crm_trace("Processing %s request sent to us from %s", op, originator);
 651         return TRUE;
 652 
 653     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 654         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
 655         *needs_reply = TRUE;
 656         return TRUE;
 657 
 658     } else if (host == NULL && cib_is_master == TRUE) {
 659         crm_trace("Processing %s request sent to master instance from %s", op, originator);
 660         return TRUE;
 661     }
 662 
 663     delegated = crm_element_value(request, F_CIB_DELEGATED);
 664     if (delegated != NULL) {
 665         crm_trace("Ignoring msg for master instance");
 666 
 667     } else if (host != NULL) {
 668         /* this is for a specific instance and we're not it */
 669         crm_trace("Ignoring msg for instance on %s", crm_str(host));
 670 
 671     } else if (reply_to == NULL && cib_is_master == FALSE) {
 672         /* this is for the master instance and we're not it */
 673         crm_trace("Ignoring reply to %s", crm_str(reply_to));
 674 
 675     } else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 676         if (reply_to != NULL) {
 677             crm_debug("Processing %s from %s", op, originator);
 678             *needs_reply = FALSE;
 679 
 680         } else {
 681             crm_debug("Processing %s reply from %s", op, originator);
 682         }
 683         return TRUE;
 684 
 685     } else {
 686         crm_err("Nothing for us to do?");
 687         crm_log_xml_err(request, "Peer[inbound]");
 688     }
 689 
 690     return FALSE;
 691 }
 692 
 693 static gboolean
 694 parse_peer_options_v2(int call_type, xmlNode * request,
     /*  */
 695                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 696                    gboolean * needs_forward)
 697 {
 698     const char *host = NULL;
 699     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
 700     const char *op = crm_element_value(request, F_CIB_OPERATION);
 701     const char *originator = crm_element_value(request, F_ORIG);
 702     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 703 
 704     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 705 
 706     if(pcmk__str_eq(op, CIB_OP_REPLACE, pcmk__str_casei)) {
 707         /* sync_our_cib() sets F_CIB_ISREPLY */
 708         if (reply_to) {
 709             delegated = reply_to;
 710         }
 711         goto skip_is_reply;
 712 
 713     } else if(pcmk__str_eq(op, CIB_OP_SYNC, pcmk__str_casei)) {
 714 
 715     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 716         process_ping_reply(request);
 717         return FALSE;
 718 
 719     } else if (pcmk__str_eq(op, CIB_OP_UPGRADE, pcmk__str_casei)) {
 720         /* Only the DC (node with the oldest software) should process
 721          * this operation if F_CIB_SCHEMA_MAX is unset
 722          *
 723          * If the DC is happy it will then send out another
 724          * CIB_OP_UPGRADE which will tell all nodes to do the actual
 725          * upgrade.
 726          *
 727          * Except this time F_CIB_SCHEMA_MAX will be set which puts a
 728          * limit on how far newer nodes will go
 729          */
 730         const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
 731         const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
 732 
 733         crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
 734                   op, (is_reply? " reply" : ""),
 735                   (cib_is_master? "master" : "slave"),
 736                   (max? max : "none"), (upgrade_rc? upgrade_rc : "none"));
 737 
 738         if (upgrade_rc != NULL) {
 739             // Our upgrade request was rejected by DC, notify clients of result
 740             crm_xml_add(request, F_CIB_RC, upgrade_rc);
 741 
 742         } else if ((max == NULL) && cib_is_master) {
 743             /* We are the DC, check if this upgrade is allowed */
 744             goto skip_is_reply;
 745 
 746         } else if(max) {
 747             /* Ok, go ahead and upgrade to 'max' */
 748             goto skip_is_reply;
 749 
 750         } else {
 751             // Ignore broadcast client requests when we're not DC
 752             return FALSE;
 753         }
 754 
 755     } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
 756         crm_info("Detected legacy %s global update from %s", op, originator);
 757         send_sync_request(NULL);
 758         legacy_mode = TRUE;
 759         return FALSE;
 760 
 761     } else if (is_reply && cib_op_modifies(call_type)) {
 762         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 763         return FALSE;
 764 
 765     } else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 766         /* Legacy handling */
 767         crm_debug("Legacy handling of %s message from %s", op, originator);
 768         *local_notify = FALSE;
 769         if (reply_to == NULL) {
 770             *process = TRUE;
 771         }
 772         return *process;
 773     }
 774 
 775     if(is_reply) {
 776         crm_trace("Handling %s reply sent from %s to local clients", op, originator);
 777         *process = FALSE;
 778         *needs_reply = FALSE;
 779         *local_notify = TRUE;
 780         return TRUE;
 781     }
 782 
 783   skip_is_reply:
 784     *process = TRUE;
 785     *needs_reply = FALSE;
 786 
 787     if(pcmk__str_eq(delegated, cib_our_uname, pcmk__str_casei)) {
 788         *local_notify = TRUE;
 789     } else {
 790         *local_notify = FALSE;
 791     }
 792 
 793     host = crm_element_value(request, F_CIB_HOST);
 794     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 795         crm_trace("Processing %s request sent to us from %s", op, originator);
 796         *needs_reply = TRUE;
 797         return TRUE;
 798 
 799     } else if (host != NULL) {
 800         /* this is for a specific instance and we're not it */
 801         crm_trace("Ignoring %s operation for instance on %s", op, crm_str(host));
 802         return FALSE;
 803 
 804     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 805         *needs_reply = TRUE;
 806     }
 807 
 808     crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
 809               crm_element_value(request, F_CIB_CLIENTNAME),
 810               crm_element_value(request, F_CIB_CALLID),
 811               originator, (*local_notify)?"(notify)":"");
 812     return TRUE;
 813 }
 814 
 815 static gboolean
 816 parse_peer_options(int call_type, xmlNode * request,
     /*  */
 817                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 818                    gboolean * needs_forward)
 819 {
 820     /* TODO: What happens when an update comes in after node A
 821      * requests the CIB from node B, but before it gets the reply (and
 822      * sends out the replace operation)
 823      */
 824     if(cib_legacy_mode()) {
 825         return parse_peer_options_v1(
 826             call_type, request, local_notify, needs_reply, process, needs_forward);
 827     } else {
 828         return parse_peer_options_v2(
 829             call_type, request, local_notify, needs_reply, process, needs_forward);
 830     }
 831 }
 832 
 833 static void
 834 forward_request(xmlNode * request, pcmk__client_t *cib_client, int call_options)
     /*  */
 835 {
 836     const char *op = crm_element_value(request, F_CIB_OPERATION);
 837     const char *host = crm_element_value(request, F_CIB_HOST);
 838 
 839     crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);
 840 
 841     if (host != NULL) {
 842         crm_trace("Forwarding %s op to %s", op, host);
 843         send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);
 844 
 845     } else {
 846         crm_trace("Forwarding %s op to master instance", op);
 847         send_cluster_message(NULL, crm_msg_cib, request, FALSE);
 848     }
 849 
 850     /* Return the request to its original state */
 851     xml_remove_prop(request, F_CIB_DELEGATED);
 852 
 853     if (call_options & cib_discard_reply) {
 854         crm_trace("Client not interested in reply");
 855     }
 856 }
 857 
 858 static gboolean
 859 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
     /*  */
 860 {
 861     CRM_ASSERT(msg != NULL);
 862 
 863     if (broadcast) {
 864         /* this (successful) call modified the CIB _and_ the
 865          * change needs to be broadcast...
 866          *   send via HA to other nodes
 867          */
 868         int diff_add_updates = 0;
 869         int diff_add_epoch = 0;
 870         int diff_add_admin_epoch = 0;
 871 
 872         int diff_del_updates = 0;
 873         int diff_del_epoch = 0;
 874         int diff_del_admin_epoch = 0;
 875 
 876         const char *digest = NULL;
 877         int format = 1;
 878 
 879         CRM_LOG_ASSERT(result_diff != NULL);
 880         digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
 881         crm_element_value_int(result_diff, "format", &format);
 882 
 883         cib_diff_version_details(result_diff,
 884                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
 885                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 886 
 887         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
 888                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
 889                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 890 
 891         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 892         pcmk__xe_set_bool_attr(msg, F_CIB_GLOBAL_UPDATE, true);
 893         crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
 894         crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
 895 
 896         if (format == 1) {
 897             CRM_ASSERT(digest != NULL);
 898         }
 899 
 900         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
 901         crm_log_xml_explicit(msg, "copy");
 902         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 903 
 904     } else if (originator != NULL) {
 905         /* send reply via HA to originating node */
 906         crm_trace("Sending request result to %s only", originator);
 907         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 908         return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
 909     }
 910 
 911     return FALSE;
 912 }
 913 
 914 /*!
 915  * \internal
 916  * \brief Handle an IPC or CPG message containing a request
 917  *
 918  * \param[in] request            Request XML
 919  * \param[in] privileged         Whether privileged commands may be run
 920  *                               (see cib_server_ops[] definition)
 921  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
 922  */
 923 static void
 924 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
 925                     pcmk__client_t *cib_client)
 926 {
 927     int call_type = 0;
 928     int call_options = 0;
 929 
 930     gboolean process = TRUE;        // Whether to process request locally now
 931     gboolean is_update = TRUE;      // Whether request would modify CIB
 932     gboolean needs_reply = TRUE;    // Whether to build a reply
 933     gboolean local_notify = FALSE;  // Whether to notify (local) requester
 934     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 935 
 936     xmlNode *op_reply = NULL;
 937     xmlNode *result_diff = NULL;
 938 
 939     int rc = pcmk_ok;
 940     const char *op = crm_element_value(request, F_CIB_OPERATION);
 941     const char *originator = crm_element_value(request, F_ORIG);
 942     const char *host = crm_element_value(request, F_CIB_HOST);
 943     const char *target = NULL;
 944     const char *call_id = crm_element_value(request, F_CIB_CALLID);
 945     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
 946     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
 947     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 948 
 949     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 950 
 951     if ((host != NULL) && (*host == '\0')) {
 952         host = NULL;
 953     }
 954 
 955     if (host) {
 956         target = host;
 957 
 958     } else if (call_options & cib_scope_local) {
 959         target = "local host";
 960 
 961     } else {
 962         target = "master";
 963     }
 964 
 965     if (cib_client == NULL) {
 966         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
 967                   op, client_name, call_id, originator, target, reply_to);
 968     } else {
 969         crm_xml_add(request, F_ORIG, cib_our_uname);
 970         crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
 971     }
 972 
 973     rc = cib_get_operation_id(op, &call_type);
 974     if (rc != pcmk_ok) {
 975         /* TODO: construct error reply? */
 976         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
 977         return;
 978     }
 979 
 980     if (cib_client != NULL) {
 981         parse_local_options(cib_client, call_type, call_options, host, op,
 982                             &local_notify, &needs_reply, &process, &needs_forward);
 983 
 984     } else if (parse_peer_options(call_type, request, &local_notify,
 985                                   &needs_reply, &process, &needs_forward) == FALSE) {
 986         return;
 987     }
 988 
 989     is_update = cib_op_modifies(call_type);
 990 
 991     if (call_options & cib_discard_reply) {
 992         /* If the request will modify the CIB, and we are in legacy mode, we
 993          * need to build a reply so we can broadcast a diff, even if the
 994          * requester doesn't want one.
 995          */
 996         needs_reply = is_update && cib_legacy_mode();
 997         local_notify = FALSE;
 998     }
 999 
1000     if (needs_forward) {
1001         const char *section = crm_element_value(request, F_CIB_SECTION);
1002         int log_level = LOG_INFO;
1003 
1004         if (pcmk__str_eq(op, CRM_OP_NOOP, pcmk__str_casei)) {
1005             log_level = LOG_DEBUG;
1006         }
1007 
1008         do_crm_log(log_level,
1009                    "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
1010                    op,
1011                    section ? section : "'all'",
1012                    host ? host : cib_legacy_mode() ? "master" : "all",
1013                    originator ? originator : "local",
1014                    client_name, call_id);
1015 
1016         forward_request(request, cib_client, call_options);
1017         return;
1018     }
1019 
1020     if (cib_status != pcmk_ok) {
1021         const char *call = crm_element_value(request, F_CIB_CALLID);
1022 
1023         rc = cib_status;
1024         crm_err("Operation ignored, cluster configuration is invalid."
1025                 " Please repair and restart: %s", pcmk_strerror(cib_status));
1026 
1027         op_reply = create_xml_node(NULL, "cib-reply");
1028         crm_xml_add(op_reply, F_TYPE, T_CIB);
1029         crm_xml_add(op_reply, F_CIB_OPERATION, op);
1030         crm_xml_add(op_reply, F_CIB_CALLID, call);
1031         crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
1032         crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
1033         crm_xml_add_int(op_reply, F_CIB_RC, rc);
1034 
1035         crm_trace("Attaching reply output");
1036         add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);
1037 
1038         crm_log_xml_explicit(op_reply, "cib:reply");
1039 
1040     } else if (process) {
1041         time_t finished = 0;
1042         time_t now = time(NULL);
1043         int level = LOG_INFO;
1044         const char *section = crm_element_value(request, F_CIB_SECTION);
1045 
1046         rc = cib_process_command(request, &op_reply, &result_diff, privileged);
1047 
1048         if (!is_update) {
1049             level = LOG_TRACE;
1050 
1051         } else if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1052             switch (rc) {
1053                 case pcmk_ok:
1054                     level = LOG_INFO;
1055                     break;
1056                 case -pcmk_err_old_data:
1057                 case -pcmk_err_diff_resync:
1058                 case -pcmk_err_diff_failed:
1059                     level = LOG_TRACE;
1060                     break;
1061                 default:
1062                     level = LOG_ERR;
1063             }
1064 
1065         } else if (rc != pcmk_ok) {
1066             level = LOG_WARNING;
1067         }
1068 
1069         do_crm_log(level,
1070                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
1071                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
1072                    originator ? originator : "local", client_name, call_id,
1073                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
1074                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
1075                    the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
1076 
1077         finished = time(NULL);
1078         if ((finished - now) > 3) {
1079             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
1080             crm_write_blackbox(0, NULL);
1081         }
1082 
1083         if (op_reply == NULL && (needs_reply || local_notify)) {
1084             crm_err("Unexpected NULL reply to message");
1085             crm_log_xml_err(request, "null reply");
1086             needs_reply = FALSE;
1087             local_notify = FALSE;
1088         }
1089     }
1090 
1091     if (is_update && !cib_legacy_mode()) {
1092         crm_trace("Completed pre-sync update from %s/%s/%s%s",
1093                   originator ? originator : "local", client_name, call_id,
1094                   local_notify?" with local notification":"");
1095 
1096     } else if (!needs_reply || stand_alone) {
1097         // This was a non-originating slave update
1098         crm_trace("Completed slave update");
1099 
1100     } else if (cib_legacy_mode() &&
1101                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
1102         gboolean broadcast = FALSE;
1103 
1104         cib_local_bcast_num++;
1105         crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
1106         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
1107 
1108         if (broadcast && client_id && local_notify && op_reply) {
1109 
1110             /* If we have been asked to sync the reply,
1111              * and a bcast msg has gone out, we queue the local notify
1112              * until we know the bcast message has been received */
1113             local_notify = FALSE;
1114             crm_trace("Queuing local %ssync notification for %s",
1115                       (call_options & cib_sync_call) ? "" : "a-", client_id);
1116 
1117             queue_local_notify(op_reply, client_id,
1118                                pcmk_is_set(call_options, cib_sync_call),
1119                                (cib_client == NULL));
1120             op_reply = NULL;    /* the reply is queued, so don't free here */
1121         }
1122 
1123     } else if (call_options & cib_discard_reply) {
1124         crm_trace("Caller isn't interested in reply");
1125 
1126     } else if (cib_client == NULL) {
1127         if (is_update == FALSE || result_diff == NULL) {
1128             crm_trace("Request not broadcast: R/O call");
1129 
1130         } else if (call_options & cib_inhibit_bcast) {
1131             crm_trace("Request not broadcast: inhibited");
1132 
1133         } else if (rc != pcmk_ok) {
1134             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
1135 
1136         } else {
1137             crm_trace("Directing reply to %s", originator);
1138         }
1139 
1140         send_peer_reply(op_reply, result_diff, originator, FALSE);
1141     }
1142 
1143     if (local_notify && client_id) {
1144         crm_trace("Performing local %ssync notification for %s",
1145                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
1146                   client_id);
1147         if (process == FALSE) {
1148             do_local_notify(request, client_id,
1149                             pcmk_is_set(call_options, cib_sync_call),
1150                             (cib_client == NULL));
1151         } else {
1152             do_local_notify(op_reply, client_id,
1153                             pcmk_is_set(call_options, cib_sync_call),
1154                             (cib_client == NULL));
1155         }
1156     }
1157 
1158     free_xml(op_reply);
1159     free_xml(result_diff);
1160 
1161     return;
1162 }
1163 
1164 static char *
1165 calculate_section_digest(const char *xpath, xmlNode * xml_obj)
     /*  */
1166 {
1167     xmlNode *xml_section = NULL;
1168 
1169     if (xml_obj == NULL) {
1170         return NULL;
1171     }
1172 
1173     xml_section = get_xpath_object(xpath, xml_obj, LOG_TRACE);
1174     if (xml_section == NULL) {
1175         return NULL;
1176     }
1177     return calculate_xml_versioned_digest(xml_section, FALSE, TRUE, CRM_FEATURE_SET); 
1178 
1179 }
1180 
1181 static int
1182 cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gboolean privileged)
     /*  */
1183 {
1184     xmlNode *input = NULL;
1185     xmlNode *output = NULL;
1186     xmlNode *result_cib = NULL;
1187     xmlNode *current_cib = NULL;
1188 
1189     int call_type = 0;
1190     int call_options = 0;
1191 
1192     const char *op = NULL;
1193     const char *section = NULL;
1194     const char *call_id = crm_element_value(request, F_CIB_CALLID);
1195 
1196     int rc = pcmk_ok;
1197     int rc2 = pcmk_ok;
1198 
1199     gboolean send_r_notify = FALSE;
1200     gboolean global_update = FALSE;
1201     gboolean config_changed = FALSE;
1202     gboolean manage_counters = TRUE;
1203 
1204     static mainloop_timer_t *digest_timer = NULL;
1205 
1206     char *current_nodes_digest = NULL;
1207     char *current_alerts_digest = NULL;
1208     char *current_status_digest = NULL;
1209     int change_section = cib_change_section_nodes | cib_change_section_alerts | cib_change_section_status;
1210 
1211     CRM_ASSERT(cib_status == pcmk_ok);
1212 
1213     if(digest_timer == NULL) {
1214         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1215     }
1216 
1217     *reply = NULL;
1218     *cib_diff = NULL;
1219     current_cib = the_cib;
1220 
1221     /* Start processing the request... */
1222     op = crm_element_value(request, F_CIB_OPERATION);
1223     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
1224     rc = cib_get_operation_id(op, &call_type);
1225 
1226     if (rc == pcmk_ok && privileged == FALSE) {
1227         rc = cib_op_can_run(call_type, call_options, privileged, global_update);
1228     }
1229 
1230     rc2 = cib_op_prepare(call_type, request, &input, &section);
1231     if (rc == pcmk_ok) {
1232         rc = rc2;
1233     }
1234 
1235     if (rc != pcmk_ok) {
1236         crm_trace("Call setup failed: %s", pcmk_strerror(rc));
1237         goto done;
1238 
1239     } else if (cib_op_modifies(call_type) == FALSE) {
1240         rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
1241                             section, request, input, FALSE, &config_changed,
1242                             current_cib, &result_cib, NULL, &output);
1243 
1244         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
1245         goto done;
1246     }
1247 
1248     /* Handle a valid write action */
1249     if (pcmk__xe_attr_is_true(request, F_CIB_GLOBAL_UPDATE)) {
1250         /* legacy code */
1251         manage_counters = FALSE;
1252         cib__set_call_options(call_options, "call", cib_force_diff);
1253         crm_trace("Global update detected");
1254 
1255         CRM_CHECK(call_type == 3 || call_type == 4, crm_err("Call type: %d", call_type);
1256                   crm_log_xml_err(request, "bad op"));
1257     }
1258 
1259     if (rc == pcmk_ok) {
1260         ping_modified_since = TRUE;
1261         if (call_options & cib_inhibit_bcast) {
1262             /* skip */
1263             crm_trace("Skipping update: inhibit broadcast");
1264             manage_counters = FALSE;
1265         }
1266 
1267         if (!pcmk_is_set(call_options, cib_dryrun)
1268             && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
1269             /* Copying large CIBs accounts for a huge percentage of our CIB usage */
1270             cib__set_call_options(call_options, "call", cib_zero_copy);
1271         } else {
1272             cib__clear_call_options(call_options, "call", cib_zero_copy);
1273         }
1274 
1275         /* Calculate the hash value of the section before the change. */
1276         if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
1277             current_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, current_cib);
1278             current_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, current_cib);
1279             current_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, current_cib);
1280             crm_trace("current-digest %s:%s:%s", current_nodes_digest, current_alerts_digest, current_status_digest);
1281         }
1282 
1283         /* result_cib must not be modified after cib_perform_op() returns */
1284         rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
1285                             section, request, input, manage_counters, &config_changed,
1286                             current_cib, &result_cib, cib_diff, &output);
1287 
1288         if (manage_counters == FALSE) {
1289             int format = 1;
1290             /* Legacy code
1291              * If the diff is NULL at this point, it's because nothing changed
1292              */
1293             if (*cib_diff) {
1294                 crm_element_value_int(*cib_diff, "format", &format);
1295             }
1296 
1297             if (format == 1) {
1298                 config_changed = cib_config_changed(NULL, NULL, cib_diff);
1299             }
1300         }
1301 
1302         /* Always write to disk for replace ops,
1303          * this also negates the need to detect ordering changes
1304          */
1305         if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
1306             config_changed = TRUE;
1307         }
1308     }
1309 
1310     if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
1311         crm_trace("Activating %s->%s%s%s",
1312                   crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
1313                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
1314                   (pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
1315                   (config_changed? " changed" : ""));
1316         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1317             rc = activateCibXml(result_cib, config_changed, op);
1318             crm_trace("Activated %s (%d)",
1319                       crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
1320         }
1321 
1322         if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
1323             cib_read_config(config_hash, result_cib);
1324         }
1325 
1326         if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
1327             char *result_nodes_digest = NULL;
1328             char *result_alerts_digest = NULL;
1329             char *result_status_digest = NULL;
1330 
1331             /* Calculate the hash value of the changed section. */
1332             result_nodes_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_NODES, result_cib);
1333             result_alerts_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION "/" XML_CIB_TAG_ALERTS, result_cib);
1334             result_status_digest = calculate_section_digest("//" XML_TAG_CIB "/" XML_CIB_TAG_STATUS, result_cib);
1335             crm_trace("result-digest %s:%s:%s", result_nodes_digest, result_alerts_digest, result_status_digest);
1336 
1337             if (pcmk__str_eq(current_nodes_digest, result_nodes_digest, pcmk__str_none)) {
1338                 change_section = change_section ^ cib_change_section_nodes;
1339             }
1340             if (pcmk__str_eq(current_alerts_digest, result_alerts_digest, pcmk__str_none)) {
1341                 change_section = change_section ^ cib_change_section_alerts;
1342             }
1343             if (pcmk__str_eq(current_status_digest, result_status_digest, pcmk__str_none)) {
1344                 change_section = change_section ^ cib_change_section_status;
1345             }
1346 
1347             if (change_section) {
1348                 send_r_notify = TRUE;
1349             }
1350             
1351             free(result_nodes_digest);
1352             free(result_alerts_digest);
1353             free(result_status_digest);
1354 
1355         } else if (pcmk__str_eq(CIB_OP_ERASE, op, pcmk__str_none)) {
1356             send_r_notify = TRUE;
1357         }
1358 
1359         mainloop_timer_stop(digest_timer);
1360         mainloop_timer_start(digest_timer);
1361 
1362     } else if (rc == -pcmk_err_schema_validation) {
1363         CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));
1364 
1365         if (output != NULL) {
1366             crm_log_xml_info(output, "cib:output");
1367             free_xml(output);
1368         }
1369 
1370         output = result_cib;
1371 
1372     } else {
1373         crm_trace("Not activating %d %d %s", rc,
1374                   pcmk_is_set(call_options, cib_dryrun),
1375                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
1376         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1377             free_xml(result_cib);
1378         }
1379     }
1380 
1381     if ((call_options & (cib_inhibit_notify|cib_dryrun)) == 0) {
1382         const char *client = crm_element_value(request, F_CIB_CLIENTNAME);
1383 
1384         crm_trace("Sending notifications %d",
1385                   pcmk_is_set(call_options, cib_dryrun));
1386         cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
1387     }
1388 
1389     if (send_r_notify) {
1390         const char *origin = crm_element_value(request, F_ORIG);
1391 
1392         cib_replace_notify(origin, the_cib, rc, *cib_diff, change_section);
1393     }
1394 
1395     xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
1396   done:
1397     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
1398         const char *caller = crm_element_value(request, F_CIB_CLIENTID);
1399 
1400         *reply = create_xml_node(NULL, "cib-reply");
1401         crm_xml_add(*reply, F_TYPE, T_CIB);
1402         crm_xml_add(*reply, F_CIB_OPERATION, op);
1403         crm_xml_add(*reply, F_CIB_CALLID, call_id);
1404         crm_xml_add(*reply, F_CIB_CLIENTID, caller);
1405         crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
1406         crm_xml_add_int(*reply, F_CIB_RC, rc);
1407 
1408         if (output != NULL) {
1409             crm_trace("Attaching reply output");
1410             add_message_xml(*reply, F_CIB_CALLDATA, output);
1411         }
1412 
1413         crm_log_xml_explicit(*reply, "cib:reply");
1414     }
1415 
1416     crm_trace("cleanup");
1417 
1418     if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
1419         free_xml(output);
1420         output = NULL;
1421     }
1422 
1423     if (call_type >= 0) {
1424         cib_op_cleanup(call_type, call_options, &input, &output);
1425     }
1426 
1427     free(current_nodes_digest);
1428     free(current_alerts_digest);
1429     free(current_status_digest);
1430 
1431     crm_trace("done");
1432     return rc;
1433 }
1434 
1435 void
1436 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1437 {
1438     const char *reason = NULL;
1439     const char *originator = crm_element_value(msg, F_ORIG);
1440 
1441     if (cib_legacy_mode() && pcmk__str_eq(originator, cib_our_uname, pcmk__str_null_matches)) {
1442         /* message is from ourselves */
1443         int bcast_id = 0;
1444 
1445         if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
1446             check_local_notify(bcast_id);
1447         }
1448         return;
1449 
1450     } else if (crm_peer_cache == NULL) {
1451         reason = "membership not established";
1452         goto bail;
1453     }
1454 
1455     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
1456         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
1457     }
1458 
1459     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1460     cib_process_request(msg, TRUE, NULL);
1461     return;
1462 
1463   bail:
1464     if (reason) {
1465         const char *seq = crm_element_value(msg, F_SEQ);
1466         const char *op = crm_element_value(msg, F_CIB_OPERATION);
1467 
1468         crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
1469     }
1470 }
1471 
1472 static gboolean
1473 cib_force_exit(gpointer data)
     /*  */
1474 {
1475     crm_notice("Forcing exit!");
1476     terminate_cib(__func__, CRM_EX_ERROR);
1477     return FALSE;
1478 }
1479 
1480 static void
1481 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1482 {
1483     pcmk__client_t *a_client = value;
1484 
1485     crm_err("Can't disconnect client %s: Not implemented",
1486             pcmk__client_name(a_client));
1487 }
1488 
1489 void
1490 cib_shutdown(int nsig)
     /*  */
1491 {
1492     struct qb_ipcs_stats srv_stats;
1493 
1494     if (cib_shutdown_flag == FALSE) {
1495         int disconnects = 0;
1496         qb_ipcs_connection_t *c = NULL;
1497 
1498         cib_shutdown_flag = TRUE;
1499 
1500         c = qb_ipcs_connection_first_get(ipcs_rw);
1501         while (c != NULL) {
1502             qb_ipcs_connection_t *last = c;
1503 
1504             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1505 
1506             crm_debug("Disconnecting r/w client %p...", last);
1507             qb_ipcs_disconnect(last);
1508             qb_ipcs_connection_unref(last);
1509             disconnects++;
1510         }
1511 
1512         c = qb_ipcs_connection_first_get(ipcs_ro);
1513         while (c != NULL) {
1514             qb_ipcs_connection_t *last = c;
1515 
1516             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1517 
1518             crm_debug("Disconnecting r/o client %p...", last);
1519             qb_ipcs_disconnect(last);
1520             qb_ipcs_connection_unref(last);
1521             disconnects++;
1522         }
1523 
1524         c = qb_ipcs_connection_first_get(ipcs_shm);
1525         while (c != NULL) {
1526             qb_ipcs_connection_t *last = c;
1527 
1528             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1529 
1530             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1531             qb_ipcs_disconnect(last);
1532             qb_ipcs_connection_unref(last);
1533             disconnects++;
1534         }
1535 
1536         disconnects += pcmk__ipc_client_count();
1537 
1538         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1539         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1540         crm_info("Disconnected %d clients", disconnects);
1541     }
1542 
1543     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1544 
1545     if (pcmk__ipc_client_count() == 0) {
1546         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1547         initiate_exit();
1548 
1549     } else {
1550         crm_info("Waiting on %d clients to disconnect (%d)",
1551                  pcmk__ipc_client_count(), srv_stats.active_connections);
1552     }
1553 }
1554 
1555 void
1556 initiate_exit(void)
     /*  */
1557 {
1558     int active = 0;
1559     xmlNode *leaving = NULL;
1560 
1561     active = crm_active_peers();
1562     if (active < 2) {
1563         terminate_cib(__func__, 0);
1564         return;
1565     }
1566 
1567     crm_info("Sending disconnect notification to %d peers...", active);
1568 
1569     leaving = create_xml_node(NULL, "exit-notification");
1570     crm_xml_add(leaving, F_TYPE, "cib");
1571     crm_xml_add(leaving, F_CIB_OPERATION, "cib_shutdown_req");
1572 
1573     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
1574     free_xml(leaving);
1575 
1576     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1577 }
1578 
1579 extern int remote_fd;
1580 extern int remote_tls_fd;
1581 
1582 /*!
1583  * \internal
1584  * \brief Close remote sockets, free the global CIB and quit
1585  *
1586  * \param[in] caller           Name of calling function (for log message)
1587  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1588  */
1589 void
1590 terminate_cib(const char *caller, int fast)
     /*  */
1591 {
1592     crm_info("%s: Exiting%s...", caller,
1593              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1594 
1595     if (remote_fd > 0) {
1596         close(remote_fd);
1597         remote_fd = 0;
1598     }
1599     if (remote_tls_fd > 0) {
1600         close(remote_tls_fd);
1601         remote_tls_fd = 0;
1602     }
1603 
1604     uninitializeCib();
1605 
1606     if (fast > 0) {
1607         /* Quit fast on error */
1608         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1609         crm_exit(fast);
1610 
1611     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1612         /* Quit via returning from the main loop. If fast == -1, we skip the
1613          * disconnect here, and it will be done when the main loop returns
1614          * (this allows the peer status callback to avoid messing with the
1615          * peer caches).
1616          */
1617         if (fast == 0) {
1618             crm_cluster_disconnect(&crm_cluster);
1619         }
1620         g_main_loop_quit(mainloop);
1621 
1622     } else {
1623         /* Quit via clean exit. Even the peer status callback can disconnect
1624          * here, because we're not returning control to the caller. */
1625         crm_cluster_disconnect(&crm_cluster);
1626         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1627         crm_exit(CRM_EX_OK);
1628     }
1629 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
