daemons/based/based

/* */
This source file includes following definitions.
cib_legacy_mode
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
do_local_notify
local_notify_destroy_callback
check_local_notify
queue_local_notify
parse_local_options_v1
parse_local_options_v2
parse_local_options
parse_peer_options_v1
parse_peer_options_v2
parse_peer_options
forward_request
send_peer_reply
cib_process_request
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
cib_shutdown
initiate_exit
terminate_cib
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>  /* U64T ~ PRIu64 */
  22 
  23 #include <crm/crm.h>
  24 #include <crm/cib.h>
  25 #include <crm/msg_xml.h>
  26 #include <crm/cluster/internal.h>
  27 
  28 #include <crm/common/xml.h>
  29 #include <crm/common/remote_internal.h>
  30 
  31 #include <pacemaker-based.h>
  32 
  33 #define EXIT_ESCALATION_MS 10000
  34 
  35 static unsigned long cib_local_bcast_num = 0;
  36 
  37 typedef struct cib_local_notify_s {
  38     xmlNode *notify_src;
  39     char *client_id;
  40     gboolean from_peer;
  41     gboolean sync_reply;
  42 } cib_local_notify_t;
  43 
  44 int next_client_id = 0;
  45 
  46 gboolean legacy_mode = FALSE;
  47 
  48 qb_ipcs_service_t *ipcs_ro = NULL;
  49 qb_ipcs_service_t *ipcs_rw = NULL;
  50 qb_ipcs_service_t *ipcs_shm = NULL;
  51 
  52 void send_cib_replace(const xmlNode * sync_request, const char *host);
  53 static void cib_process_request(xmlNode *request, gboolean privileged,
  54                                 pcmk__client_t *cib_client);
  55 
  56 
  57 static int cib_process_command(xmlNode *request, xmlNode **reply,
  58                                xmlNode **cib_diff, gboolean privileged);
  59 
  60 gboolean cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size,
  61                              gboolean privileged);
  62 
  63 gboolean cib_legacy_mode(void)
     /*  */
  64 {
  65     return legacy_mode;
  66 }
  67 
  68 
  69 static int32_t
  70 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  71 {
  72     if (cib_shutdown_flag) {
  73         crm_info("Ignoring new IPC client [%d] during shutdown",
  74                  pcmk__client_pid(c));
  75         return -EPERM;
  76     }
  77 
  78     if (pcmk__new_client(c, uid, gid) == NULL) {
  79         return -EIO;
  80     }
  81     return 0;
  82 }
  83 
  84 static int32_t
  85 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  86 {
  87     pcmk__client_t *client = pcmk__find_client(c);
  88 
  89     crm_trace("%p message from %s", c, client->id);
  90     return cib_common_callback(c, data, size, TRUE);
  91 }
  92 
  93 static int32_t
  94 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  95 {
  96     pcmk__client_t *client = pcmk__find_client(c);
  97 
  98     crm_trace("%p message from %s", c, client->id);
  99     return cib_common_callback(c, data, size, FALSE);
 100 }
 101 
 102 /* Error code means? */
 103 static int32_t
 104 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 105 {
 106     pcmk__client_t *client = pcmk__find_client(c);
 107 
 108     if (client == NULL) {
 109         return 0;
 110     }
 111     crm_trace("Connection %p", c);
 112     pcmk__free_client(client);
 113     return 0;
 114 }
 115 
 116 static void
 117 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 118 {
 119     crm_trace("Connection %p", c);
 120     cib_ipc_closed(c);
 121     if (cib_shutdown_flag) {
 122         cib_shutdown(0);
 123     }
 124 }
 125 
 126 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 127     .connection_accept = cib_ipc_accept,
 128     .connection_created = NULL,
 129     .msg_process = cib_ipc_dispatch_ro,
 130     .connection_closed = cib_ipc_closed,
 131     .connection_destroyed = cib_ipc_destroy
 132 };
 133 
 134 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 135     .connection_accept = cib_ipc_accept,
 136     .connection_created = NULL,
 137     .msg_process = cib_ipc_dispatch_rw,
 138     .connection_closed = cib_ipc_closed,
 139     .connection_destroyed = cib_ipc_destroy
 140 };
 141 
 142 void
 143 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 144                            pcmk__client_t *cib_client, gboolean privileged)
 145 {
 146     const char *op = crm_element_value(op_request, F_CIB_OPERATION);
 147 
 148     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 149         if (flags & crm_ipc_client_response) {
 150             xmlNode *ack = create_xml_node(NULL, __func__);
 151 
 152             crm_xml_add(ack, F_CIB_OPERATION, CRM_OP_REGISTER);
 153             crm_xml_add(ack, F_CIB_CLIENTID, cib_client->id);
 154             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 155             cib_client->request_id = 0;
 156             free_xml(ack);
 157         }
 158         return;
 159 
 160     } else if (pcmk__str_eq(op, T_CIB_NOTIFY, pcmk__str_none)) {
 161         /* Update the notify filters for this client */
 162         int on_off = 0;
 163         crm_exit_t status = CRM_EX_OK;
 164         uint64_t bit = UINT64_C(0);
 165         const char *type = crm_element_value(op_request, F_CIB_NOTIFY_TYPE);
 166 
 167         crm_element_value_int(op_request, F_CIB_NOTIFY_ACTIVATE, &on_off);
 168 
 169         crm_debug("Setting %s callbacks %s for client %s",
 170                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 171 
 172         if (pcmk__str_eq(type, T_CIB_POST_NOTIFY, pcmk__str_casei)) {
 173             bit = cib_notify_post;
 174 
 175         } else if (pcmk__str_eq(type, T_CIB_PRE_NOTIFY, pcmk__str_casei)) {
 176             bit = cib_notify_pre;
 177 
 178         } else if (pcmk__str_eq(type, T_CIB_UPDATE_CONFIRM, pcmk__str_casei)) {
 179             bit = cib_notify_confirm;
 180 
 181         } else if (pcmk__str_eq(type, T_CIB_DIFF_NOTIFY, pcmk__str_casei)) {
 182             bit = cib_notify_diff;
 183 
 184         } else if (pcmk__str_eq(type, T_CIB_REPLACE_NOTIFY, pcmk__str_casei)) {
 185             bit = cib_notify_replace;
 186 
 187         } else {
 188             status = CRM_EX_INVALID_PARAM;
 189         }
 190 
 191         if (bit != 0) {
 192             if (on_off) {
 193                 pcmk__set_client_flags(cib_client, bit);
 194             } else {
 195                 pcmk__clear_client_flags(cib_client, bit);
 196             }
 197         }
 198 
 199         pcmk__ipc_send_ack(cib_client, id, flags, "ack", status);
 200         return;
 201     }
 202 
 203     cib_process_request(op_request, privileged, cib_client);
 204 }
 205 
 206 int32_t
 207 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 208 {
 209     uint32_t id = 0;
 210     uint32_t flags = 0;
 211     int call_options = 0;
 212     pcmk__client_t *cib_client = pcmk__find_client(c);
 213     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 214 
 215     if (op_request) {
 216         crm_element_value_int(op_request, F_CIB_CALLOPTS, &call_options);
 217     }
 218 
 219     if (op_request == NULL) {
 220         crm_trace("Invalid message from %p", c);
 221         pcmk__ipc_send_ack(cib_client, id, flags, "nack", CRM_EX_PROTOCOL);
 222         return 0;
 223 
 224     } else if(cib_client == NULL) {
 225         crm_trace("Invalid client %p", c);
 226         return 0;
 227     }
 228 
 229     if (pcmk_is_set(call_options, cib_sync_call)) {
 230         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 231         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 232         cib_client->request_id = id;    /* Reply only to the last one */
 233     }
 234 
 235     if (cib_client->name == NULL) {
 236         const char *value = crm_element_value(op_request, F_CIB_CLIENTNAME);
 237 
 238         if (value == NULL) {
 239             cib_client->name = pcmk__itoa(cib_client->pid);
 240         } else {
 241             cib_client->name = strdup(value);
 242             if (crm_is_daemon_name(value)) {
 243                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 244             }
 245         }
 246     }
 247 
 248     /* Allow cluster daemons more leeway before being evicted */
 249     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 250         const char *qmax = cib_config_lookup("cluster-ipc-limit");
 251 
 252         if (pcmk__set_client_queue_max(cib_client, qmax)) {
 253             crm_trace("IPC threshold for client %s[%u] is now %u",
 254                       pcmk__client_name(cib_client), cib_client->pid,
 255                       cib_client->queue_max);
 256         }
 257     }
 258 
 259     crm_xml_add(op_request, F_CIB_CLIENTID, cib_client->id);
 260     crm_xml_add(op_request, F_CIB_CLIENTNAME, cib_client->name);
 261 
 262     CRM_LOG_ASSERT(cib_client->user != NULL);
 263     pcmk__update_acl_user(op_request, F_CIB_USER, cib_client->user);
 264 
 265     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 266     free_xml(op_request);
 267 
 268     return 0;
 269 }
 270 
 271 static uint64_t ping_seq = 0;
 272 static char *ping_digest = NULL;
 273 static bool ping_modified_since = FALSE;
 274 int sync_our_cib(xmlNode * request, gboolean all);
 275 
 276 static gboolean
 277 cib_digester_cb(gpointer data)
     /*  */
 278 {
 279     if (cib_is_master) {
 280         char buffer[32];
 281         xmlNode *ping = create_xml_node(NULL, "ping");
 282 
 283         ping_seq++;
 284         free(ping_digest);
 285         ping_digest = NULL;
 286         ping_modified_since = FALSE;
 287         snprintf(buffer, 32, "%" U64T, ping_seq);
 288         crm_trace("Requesting peer digests (%s)", buffer);
 289 
 290         crm_xml_add(ping, F_TYPE, "cib");
 291         crm_xml_add(ping, F_CIB_OPERATION, CRM_OP_PING);
 292         crm_xml_add(ping, F_CIB_PING_ID, buffer);
 293 
 294         crm_xml_add(ping, XML_ATTR_CRM_VERSION, CRM_FEATURE_SET);
 295         send_cluster_message(NULL, crm_msg_cib, ping, TRUE);
 296 
 297         free_xml(ping);
 298     }
 299     return FALSE;
 300 }
 301 
 302 static void
 303 process_ping_reply(xmlNode *reply) 
     /*  */
 304 {
 305     uint64_t seq = 0;
 306     const char *host = crm_element_value(reply, F_ORIG);
 307 
 308     xmlNode *pong = get_message_xml(reply, F_CIB_CALLDATA);
 309     const char *seq_s = crm_element_value(pong, F_CIB_PING_ID);
 310     const char *digest = crm_element_value(pong, XML_ATTR_DIGEST);
 311 
 312     if (seq_s == NULL) {
 313         crm_debug("Ignoring ping reply with no " F_CIB_PING_ID);
 314         return;
 315 
 316     } else {
 317         long long seq_ll;
 318 
 319         if (pcmk__scan_ll(seq_s, &seq_ll, 0LL) != pcmk_rc_ok) {
 320             return;
 321         }
 322         seq = (uint64_t) seq_ll;
 323     }
 324 
 325     if(digest == NULL) {
 326         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 327 
 328     } else if(seq != ping_seq) {
 329         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 330 
 331     } else if(ping_modified_since) {
 332         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 333 
 334     } else {
 335         const char *version = crm_element_value(pong, XML_ATTR_CRM_VERSION);
 336 
 337         if(ping_digest == NULL) {
 338             crm_trace("Calculating new digest");
 339             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
 340         }
 341 
 342         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 343         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 344             xmlNode *remote_cib = get_message_xml(pong, F_CIB_CALLDATA);
 345 
 346             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 347                        crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN),
 348                        crm_element_value(the_cib, XML_ATTR_GENERATION),
 349                        crm_element_value(the_cib, XML_ATTR_NUMUPDATES),
 350                        ping_digest, host,
 351                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION_ADMIN):"_",
 352                        remote_cib?crm_element_value(remote_cib, XML_ATTR_GENERATION):"_",
 353                        remote_cib?crm_element_value(remote_cib, XML_ATTR_NUMUPDATES):"_",
 354                        digest, remote_cib);
 355 
 356             if(remote_cib && remote_cib->children) {
 357                 /* Additional debug */
 358                 xml_calculate_changes(the_cib, remote_cib);
 359                 xml_log_changes(LOG_INFO, __func__, remote_cib);
 360                 crm_trace("End of differences");
 361             }
 362 
 363             free_xml(remote_cib);
 364             sync_our_cib(reply, FALSE);
 365         }
 366     }
 367 }
 368 
 369 static void
 370 do_local_notify(xmlNode * notify_src, const char *client_id,
     /*  */
 371                 gboolean sync_reply, gboolean from_peer)
 372 {
 373     int rid = 0;
 374     int call_id = 0;
 375     pcmk__client_t *client_obj = NULL;
 376 
 377     CRM_ASSERT(notify_src && client_id);
 378 
 379     crm_element_value_int(notify_src, F_CIB_CALLID, &call_id);
 380 
 381     client_obj = pcmk__find_client_by_id(client_id);
 382     if (client_obj == NULL) {
 383         crm_debug("Could not send response %d: client %s not found",
 384                   call_id, client_id);
 385         return;
 386     }
 387 
 388     if (sync_reply) {
 389         if (client_obj->ipcs) {
 390             CRM_LOG_ASSERT(client_obj->request_id);
 391 
 392             rid = client_obj->request_id;
 393             client_obj->request_id = 0;
 394 
 395             crm_trace("Sending response %d to client %s%s",
 396                       rid, pcmk__client_name(client_obj),
 397                       (from_peer? " (originator of delegated request)" : ""));
 398         } else {
 399             crm_trace("Sending response (call %d) to client %s%s",
 400                       call_id, pcmk__client_name(client_obj),
 401                       (from_peer? " (originator of delegated request)" : ""));
 402         }
 403 
 404     } else {
 405         crm_trace("Sending event %d to client %s%s",
 406                   call_id, pcmk__client_name(client_obj),
 407                   (from_peer? " (originator of delegated request)" : ""));
 408     }
 409 
 410     switch (PCMK__CLIENT_TYPE(client_obj)) {
 411         case pcmk__client_ipc:
 412             {
 413                 int rc = pcmk__ipc_send_xml(client_obj, rid, notify_src,
 414                                             (sync_reply? crm_ipc_flags_none
 415                                              : crm_ipc_server_event));
 416 
 417                 if (rc != pcmk_rc_ok) {
 418                     crm_warn("%s reply to client %s failed: %s " CRM_XS " rc=%d",
 419                              (sync_reply? "Synchronous" : "Asynchronous"),
 420                              pcmk__client_name(client_obj), pcmk_rc_str(rc),
 421                              rc);
 422                 }
 423             }
 424             break;
 425 #ifdef HAVE_GNUTLS_GNUTLS_H
 426         case pcmk__client_tls:
 427 #endif
 428         case pcmk__client_tcp:
 429             pcmk__remote_send_xml(client_obj->remote, notify_src);
 430             break;
 431         default:
 432             crm_err("Unknown transport for client %s "
 433                     CRM_XS " flags=0x%016" PRIx64,
 434                     pcmk__client_name(client_obj), client_obj->flags);
 435     }
 436 }
 437 
 438 static void
 439 local_notify_destroy_callback(gpointer data)
     /*  */
 440 {
 441     cib_local_notify_t *notify = data;
 442 
 443     free_xml(notify->notify_src);
 444     free(notify->client_id);
 445     free(notify);
 446 }
 447 
 448 static void
 449 check_local_notify(int bcast_id)
     /*  */
 450 {
 451     cib_local_notify_t *notify = NULL;
 452 
 453     if (!local_notify_queue) {
 454         return;
 455     }
 456 
 457     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 458 
 459     if (notify) {
 460         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
 461                         notify->from_peer);
 462         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
 463     }
 464 }
 465 
 466 static void
 467 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
     /*  */
 468                    gboolean from_peer)
 469 {
 470     cib_local_notify_t *notify = calloc(1, sizeof(cib_local_notify_t));
 471 
 472     notify->notify_src = notify_src;
 473     notify->client_id = strdup(client_id);
 474     notify->sync_reply = sync_reply;
 475     notify->from_peer = from_peer;
 476 
 477     if (!local_notify_queue) {
 478         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
 479     }
 480     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
 481     // cppcheck doesn't know notify will get freed when hash table is destroyed
 482     // cppcheck-suppress memleak
 483 }
 484 
 485 static void
 486 parse_local_options_v1(pcmk__client_t *cib_client, int call_type,
     /*  */
 487                        int call_options, const char *host, const char *op,
 488                        gboolean *local_notify, gboolean *needs_reply,
 489                        gboolean *process, gboolean *needs_forward)
 490 {
 491     if (cib_op_modifies(call_type)
 492         && !(call_options & cib_inhibit_bcast)) {
 493         /* we need to send an update anyway */
 494         *needs_reply = TRUE;
 495     } else {
 496         *needs_reply = FALSE;
 497     }
 498 
 499     if (host == NULL && (call_options & cib_scope_local)) {
 500         crm_trace("Processing locally scoped %s op from client %s",
 501                   op, pcmk__client_name(cib_client));
 502         *local_notify = TRUE;
 503 
 504     } else if (host == NULL && cib_is_master) {
 505         crm_trace("Processing master %s op locally from client %s",
 506                   op, pcmk__client_name(cib_client));
 507         *local_notify = TRUE;
 508 
 509     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 510         crm_trace("Processing locally addressed %s op from client %s",
 511                   op, pcmk__client_name(cib_client));
 512         *local_notify = TRUE;
 513 
 514     } else if (stand_alone) {
 515         *needs_forward = FALSE;
 516         *local_notify = TRUE;
 517         *process = TRUE;
 518 
 519     } else {
 520         crm_trace("%s op from %s needs to be forwarded to client %s",
 521                   op, pcmk__client_name(cib_client),
 522                   (host? host : "the master instance"));
 523         *needs_forward = TRUE;
 524         *process = FALSE;
 525     }
 526 }
 527 
 528 static void
 529 parse_local_options_v2(pcmk__client_t *cib_client, int call_type,
     /*  */
 530                        int call_options, const char *host, const char *op,
 531                        gboolean *local_notify, gboolean *needs_reply,
 532                        gboolean *process, gboolean *needs_forward)
 533 {
 534     if (cib_op_modifies(call_type)) {
 535         if (pcmk__strcase_any_of(op, CIB_OP_MASTER, CIB_OP_SLAVE, NULL)) {
 536             /* Always handle these locally */
 537             *process = TRUE;
 538             *needs_reply = FALSE;
 539             *local_notify = TRUE;
 540             *needs_forward = FALSE;
 541             return;
 542 
 543         } else {
 544             /* Redirect all other updates via CPG */
 545             *needs_reply = TRUE;
 546             *needs_forward = TRUE;
 547             *process = FALSE;
 548             crm_trace("%s op from %s needs to be forwarded to client %s",
 549                       op, pcmk__client_name(cib_client),
 550                       (host? host : "the master instance"));
 551             return;
 552         }
 553     }
 554 
 555 
 556     *process = TRUE;
 557     *needs_reply = FALSE;
 558     *local_notify = TRUE;
 559     *needs_forward = FALSE;
 560 
 561     if (stand_alone) {
 562         crm_trace("Processing %s op from client %s (stand-alone)",
 563                   op, pcmk__client_name(cib_client));
 564 
 565     } else if (host == NULL) {
 566         crm_trace("Processing unaddressed %s op from client %s",
 567                   op, pcmk__client_name(cib_client));
 568 
 569     } else if (pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 570         crm_trace("Processing locally addressed %s op from client %s",
 571                   op, pcmk__client_name(cib_client));
 572 
 573     } else {
 574         crm_trace("%s op from %s needs to be forwarded to client %s",
 575                   op, pcmk__client_name(cib_client), host);
 576         *needs_forward = TRUE;
 577         *process = FALSE;
 578     }
 579 }
 580 
 581 static void
 582 parse_local_options(pcmk__client_t *cib_client, int call_type,
     /*  */
 583                     int call_options, const char *host, const char *op,
 584                     gboolean *local_notify, gboolean *needs_reply,
 585                     gboolean *process, gboolean *needs_forward)
 586 {
 587     if(cib_legacy_mode()) {
 588         parse_local_options_v1(cib_client, call_type, call_options, host,
 589                                op, local_notify, needs_reply, process, needs_forward);
 590     } else {
 591         parse_local_options_v2(cib_client, call_type, call_options, host,
 592                                op, local_notify, needs_reply, process, needs_forward);
 593     }
 594 }
 595 
 596 static gboolean
 597 parse_peer_options_v1(int call_type, xmlNode * request,
     /*  */
 598                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 599                    gboolean * needs_forward)
 600 {
 601     const char *op = NULL;
 602     const char *host = NULL;
 603     const char *delegated = NULL;
 604     const char *originator = crm_element_value(request, F_ORIG);
 605     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 606     const char *update = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 607 
 608     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 609 
 610     if (crm_is_true(update)) {
 611         *needs_reply = FALSE;
 612         if (is_reply) {
 613             *local_notify = TRUE;
 614             crm_trace("Processing global/peer update from %s"
 615                       " that originated from us", originator);
 616         } else {
 617             crm_trace("Processing global/peer update from %s", originator);
 618         }
 619         return TRUE;
 620     }
 621 
 622     op = crm_element_value(request, F_CIB_OPERATION);
 623     crm_trace("Processing %s request sent by %s", op, originator);
 624     if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 625         /* Always process these */
 626         *local_notify = FALSE;
 627         if (reply_to == NULL || is_reply) {
 628             *process = TRUE;
 629         }
 630         if (is_reply) {
 631             *needs_reply = FALSE;
 632         }
 633         return *process;
 634     }
 635 
 636     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 637         process_ping_reply(request);
 638         return FALSE;
 639     }
 640 
 641     if (is_reply) {
 642         crm_trace("Forward reply sent from %s to local clients", originator);
 643         *process = FALSE;
 644         *needs_reply = FALSE;
 645         *local_notify = TRUE;
 646         return TRUE;
 647     }
 648 
 649     host = crm_element_value(request, F_CIB_HOST);
 650     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 651         crm_trace("Processing %s request sent to us from %s", op, originator);
 652         return TRUE;
 653 
 654     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 655         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
 656         *needs_reply = TRUE;
 657         return TRUE;
 658 
 659     } else if (host == NULL && cib_is_master == TRUE) {
 660         crm_trace("Processing %s request sent to master instance from %s", op, originator);
 661         return TRUE;
 662     }
 663 
 664     delegated = crm_element_value(request, F_CIB_DELEGATED);
 665     if (delegated != NULL) {
 666         crm_trace("Ignoring msg for master instance");
 667 
 668     } else if (host != NULL) {
 669         /* this is for a specific instance and we're not it */
 670         crm_trace("Ignoring msg for instance on %s", crm_str(host));
 671 
 672     } else if (reply_to == NULL && cib_is_master == FALSE) {
 673         /* this is for the master instance and we're not it */
 674         crm_trace("Ignoring reply to %s", crm_str(reply_to));
 675 
 676     } else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 677         if (reply_to != NULL) {
 678             crm_debug("Processing %s from %s", op, originator);
 679             *needs_reply = FALSE;
 680 
 681         } else {
 682             crm_debug("Processing %s reply from %s", op, originator);
 683         }
 684         return TRUE;
 685 
 686     } else {
 687         crm_err("Nothing for us to do?");
 688         crm_log_xml_err(request, "Peer[inbound]");
 689     }
 690 
 691     return FALSE;
 692 }
 693 
 694 static gboolean
 695 parse_peer_options_v2(int call_type, xmlNode * request,
     /*  */
 696                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 697                    gboolean * needs_forward)
 698 {
 699     const char *host = NULL;
 700     const char *delegated = crm_element_value(request, F_CIB_DELEGATED);
 701     const char *op = crm_element_value(request, F_CIB_OPERATION);
 702     const char *originator = crm_element_value(request, F_ORIG);
 703     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 704     const char *update = crm_element_value(request, F_CIB_GLOBAL_UPDATE);
 705 
 706     gboolean is_reply = pcmk__str_eq(reply_to, cib_our_uname, pcmk__str_casei);
 707 
 708     if(pcmk__str_eq(op, CIB_OP_REPLACE, pcmk__str_casei)) {
 709         /* sync_our_cib() sets F_CIB_ISREPLY */
 710         if (reply_to) {
 711             delegated = reply_to;
 712         }
 713         goto skip_is_reply;
 714 
 715     } else if(pcmk__str_eq(op, CIB_OP_SYNC, pcmk__str_casei)) {
 716 
 717     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 718         process_ping_reply(request);
 719         return FALSE;
 720 
 721     } else if (pcmk__str_eq(op, CIB_OP_UPGRADE, pcmk__str_casei)) {
 722         /* Only the DC (node with the oldest software) should process
 723          * this operation if F_CIB_SCHEMA_MAX is unset
 724          *
 725          * If the DC is happy it will then send out another
 726          * CIB_OP_UPGRADE which will tell all nodes to do the actual
 727          * upgrade.
 728          *
 729          * Except this time F_CIB_SCHEMA_MAX will be set which puts a
 730          * limit on how far newer nodes will go
 731          */
 732         const char *max = crm_element_value(request, F_CIB_SCHEMA_MAX);
 733         const char *upgrade_rc = crm_element_value(request, F_CIB_UPGRADE_RC);
 734 
 735         crm_trace("Parsing %s operation%s for %s with max=%s and upgrade_rc=%s",
 736                   op, (is_reply? " reply" : ""),
 737                   (cib_is_master? "master" : "slave"),
 738                   (max? max : "none"), (upgrade_rc? upgrade_rc : "none"));
 739 
 740         if (upgrade_rc != NULL) {
 741             // Our upgrade request was rejected by DC, notify clients of result
 742             crm_xml_add(request, F_CIB_RC, upgrade_rc);
 743 
 744         } else if ((max == NULL) && cib_is_master) {
 745             /* We are the DC, check if this upgrade is allowed */
 746             goto skip_is_reply;
 747 
 748         } else if(max) {
 749             /* Ok, go ahead and upgrade to 'max' */
 750             goto skip_is_reply;
 751 
 752         } else {
 753             // Ignore broadcast client requests when we're not DC
 754             return FALSE;
 755         }
 756 
 757     } else if (crm_is_true(update)) {
 758         crm_info("Detected legacy %s global update from %s", op, originator);
 759         send_sync_request(NULL);
 760         legacy_mode = TRUE;
 761         return FALSE;
 762 
 763     } else if (is_reply && cib_op_modifies(call_type)) {
 764         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 765         return FALSE;
 766 
 767     } else if (pcmk__str_eq(op, "cib_shutdown_req", pcmk__str_casei)) {
 768         /* Legacy handling */
 769         crm_debug("Legacy handling of %s message from %s", op, originator);
 770         *local_notify = FALSE;
 771         if (reply_to == NULL) {
 772             *process = TRUE;
 773         }
 774         return *process;
 775     }
 776 
 777     if(is_reply) {
 778         crm_trace("Handling %s reply sent from %s to local clients", op, originator);
 779         *process = FALSE;
 780         *needs_reply = FALSE;
 781         *local_notify = TRUE;
 782         return TRUE;
 783     }
 784 
 785   skip_is_reply:
 786     *process = TRUE;
 787     *needs_reply = FALSE;
 788 
 789     if(pcmk__str_eq(delegated, cib_our_uname, pcmk__str_casei)) {
 790         *local_notify = TRUE;
 791     } else {
 792         *local_notify = FALSE;
 793     }
 794 
 795     host = crm_element_value(request, F_CIB_HOST);
 796     if (host != NULL && pcmk__str_eq(host, cib_our_uname, pcmk__str_casei)) {
 797         crm_trace("Processing %s request sent to us from %s", op, originator);
 798         *needs_reply = TRUE;
 799         return TRUE;
 800 
 801     } else if (host != NULL) {
 802         /* this is for a specific instance and we're not it */
 803         crm_trace("Ignoring %s operation for instance on %s", op, crm_str(host));
 804         return FALSE;
 805 
 806     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 807         *needs_reply = TRUE;
 808     }
 809 
 810     crm_trace("Processing %s request sent to everyone by %s/%s on %s %s", op,
 811               crm_element_value(request, F_CIB_CLIENTNAME),
 812               crm_element_value(request, F_CIB_CALLID),
 813               originator, (*local_notify)?"(notify)":"");
 814     return TRUE;
 815 }
 816 
 817 static gboolean
 818 parse_peer_options(int call_type, xmlNode * request,
     /*  */
 819                    gboolean * local_notify, gboolean * needs_reply, gboolean * process,
 820                    gboolean * needs_forward)
 821 {
 822     /* TODO: What happens when an update comes in after node A
 823      * requests the CIB from node B, but before it gets the reply (and
 824      * sends out the replace operation)
 825      */
 826     if(cib_legacy_mode()) {
 827         return parse_peer_options_v1(
 828             call_type, request, local_notify, needs_reply, process, needs_forward);
 829     } else {
 830         return parse_peer_options_v2(
 831             call_type, request, local_notify, needs_reply, process, needs_forward);
 832     }
 833 }
 834 
 835 static void
 836 forward_request(xmlNode * request, pcmk__client_t *cib_client, int call_options)
     /*  */
 837 {
 838     const char *op = crm_element_value(request, F_CIB_OPERATION);
 839     const char *host = crm_element_value(request, F_CIB_HOST);
 840 
 841     crm_xml_add(request, F_CIB_DELEGATED, cib_our_uname);
 842 
 843     if (host != NULL) {
 844         crm_trace("Forwarding %s op to %s", op, host);
 845         send_cluster_message(crm_get_peer(0, host), crm_msg_cib, request, FALSE);
 846 
 847     } else {
 848         crm_trace("Forwarding %s op to master instance", op);
 849         send_cluster_message(NULL, crm_msg_cib, request, FALSE);
 850     }
 851 
 852     /* Return the request to its original state */
 853     xml_remove_prop(request, F_CIB_DELEGATED);
 854 
 855     if (call_options & cib_discard_reply) {
 856         crm_trace("Client not interested in reply");
 857     }
 858 }
 859 
 860 static gboolean
 861 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
     /*  */
 862 {
 863     CRM_ASSERT(msg != NULL);
 864 
 865     if (broadcast) {
 866         /* this (successful) call modified the CIB _and_ the
 867          * change needs to be broadcast...
 868          *   send via HA to other nodes
 869          */
 870         int diff_add_updates = 0;
 871         int diff_add_epoch = 0;
 872         int diff_add_admin_epoch = 0;
 873 
 874         int diff_del_updates = 0;
 875         int diff_del_epoch = 0;
 876         int diff_del_admin_epoch = 0;
 877 
 878         const char *digest = NULL;
 879         int format = 1;
 880 
 881         CRM_LOG_ASSERT(result_diff != NULL);
 882         digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
 883         crm_element_value_int(result_diff, "format", &format);
 884 
 885         cib_diff_version_details(result_diff,
 886                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
 887                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
 888 
 889         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
 890                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
 891                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
 892 
 893         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 894         crm_xml_add(msg, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
 895         crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
 896         crm_xml_add(msg, F_CIB_USER, CRM_DAEMON_USER);
 897 
 898         if (format == 1) {
 899             CRM_ASSERT(digest != NULL);
 900         }
 901 
 902         add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
 903         crm_log_xml_explicit(msg, "copy");
 904         return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
 905 
 906     } else if (originator != NULL) {
 907         /* send reply via HA to originating node */
 908         crm_trace("Sending request result to %s only", originator);
 909         crm_xml_add(msg, F_CIB_ISREPLY, originator);
 910         return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
 911     }
 912 
 913     return FALSE;
 914 }
 915 
 916 /*!
 917  * \internal
 918  * \brief Handle an IPC or CPG message containing a request
 919  *
 920  * \param[in] request            Request XML
 921  * \param[in] privileged         Whether privileged commands may be run
 922  *                               (see cib_server_ops[] definition)
 923  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
 924  */
 925 static void
 926 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
 927                     pcmk__client_t *cib_client)
 928 {
 929     int call_type = 0;
 930     int call_options = 0;
 931 
 932     gboolean process = TRUE;        // Whether to process request locally now
 933     gboolean is_update = TRUE;      // Whether request would modify CIB
 934     gboolean needs_reply = TRUE;    // Whether to build a reply
 935     gboolean local_notify = FALSE;  // Whether to notify (local) requester
 936     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
 937     gboolean global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
 938 
 939     xmlNode *op_reply = NULL;
 940     xmlNode *result_diff = NULL;
 941 
 942     int rc = pcmk_ok;
 943     const char *op = crm_element_value(request, F_CIB_OPERATION);
 944     const char *originator = crm_element_value(request, F_ORIG);
 945     const char *host = crm_element_value(request, F_CIB_HOST);
 946     const char *target = NULL;
 947     const char *call_id = crm_element_value(request, F_CIB_CALLID);
 948     const char *client_id = crm_element_value(request, F_CIB_CLIENTID);
 949     const char *client_name = crm_element_value(request, F_CIB_CLIENTNAME);
 950     const char *reply_to = crm_element_value(request, F_CIB_ISREPLY);
 951 
 952     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
 953 
 954     if ((host != NULL) && (*host == '\0')) {
 955         host = NULL;
 956     }
 957 
 958     if (host) {
 959         target = host;
 960 
 961     } else if (call_options & cib_scope_local) {
 962         target = "local host";
 963 
 964     } else {
 965         target = "master";
 966     }
 967 
 968     if (cib_client == NULL) {
 969         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
 970                   op, client_name, call_id, originator, target, reply_to);
 971     } else {
 972         crm_xml_add(request, F_ORIG, cib_our_uname);
 973         crm_trace("Processing local %s operation from %s/%s intended for %s", op, client_name, call_id, target);
 974     }
 975 
 976     rc = cib_get_operation_id(op, &call_type);
 977     if (rc != pcmk_ok) {
 978         /* TODO: construct error reply? */
 979         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
 980         return;
 981     }
 982 
 983     if (cib_client != NULL) {
 984         parse_local_options(cib_client, call_type, call_options, host, op,
 985                             &local_notify, &needs_reply, &process, &needs_forward);
 986 
 987     } else if (parse_peer_options(call_type, request, &local_notify,
 988                                   &needs_reply, &process, &needs_forward) == FALSE) {
 989         return;
 990     }
 991 
 992     is_update = cib_op_modifies(call_type);
 993 
 994     if (call_options & cib_discard_reply) {
 995         /* If the request will modify the CIB, and we are in legacy mode, we
 996          * need to build a reply so we can broadcast a diff, even if the
 997          * requester doesn't want one.
 998          */
 999         needs_reply = is_update && cib_legacy_mode();
1000         local_notify = FALSE;
1001     }
1002 
1003     if (needs_forward) {
1004         const char *section = crm_element_value(request, F_CIB_SECTION);
1005         int log_level = LOG_INFO;
1006 
1007         if (pcmk__str_eq(op, CRM_OP_NOOP, pcmk__str_casei)) {
1008             log_level = LOG_DEBUG;
1009         }
1010 
1011         do_crm_log(log_level,
1012                    "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
1013                    op,
1014                    section ? section : "'all'",
1015                    host ? host : cib_legacy_mode() ? "master" : "all",
1016                    originator ? originator : "local",
1017                    client_name, call_id);
1018 
1019         forward_request(request, cib_client, call_options);
1020         return;
1021     }
1022 
1023     if (cib_status != pcmk_ok) {
1024         const char *call = crm_element_value(request, F_CIB_CALLID);
1025 
1026         rc = cib_status;
1027         crm_err("Operation ignored, cluster configuration is invalid."
1028                 " Please repair and restart: %s", pcmk_strerror(cib_status));
1029 
1030         op_reply = create_xml_node(NULL, "cib-reply");
1031         crm_xml_add(op_reply, F_TYPE, T_CIB);
1032         crm_xml_add(op_reply, F_CIB_OPERATION, op);
1033         crm_xml_add(op_reply, F_CIB_CALLID, call);
1034         crm_xml_add(op_reply, F_CIB_CLIENTID, client_id);
1035         crm_xml_add_int(op_reply, F_CIB_CALLOPTS, call_options);
1036         crm_xml_add_int(op_reply, F_CIB_RC, rc);
1037 
1038         crm_trace("Attaching reply output");
1039         add_message_xml(op_reply, F_CIB_CALLDATA, the_cib);
1040 
1041         crm_log_xml_explicit(op_reply, "cib:reply");
1042 
1043     } else if (process) {
1044         time_t finished = 0;
1045         time_t now = time(NULL);
1046         int level = LOG_INFO;
1047         const char *section = crm_element_value(request, F_CIB_SECTION);
1048 
1049         rc = cib_process_command(request, &op_reply, &result_diff, privileged);
1050 
1051         if (!is_update) {
1052             level = LOG_TRACE;
1053 
1054         } else if (global_update) {
1055             switch (rc) {
1056                 case pcmk_ok:
1057                     level = LOG_INFO;
1058                     break;
1059                 case -pcmk_err_old_data:
1060                 case -pcmk_err_diff_resync:
1061                 case -pcmk_err_diff_failed:
1062                     level = LOG_TRACE;
1063                     break;
1064                 default:
1065                     level = LOG_ERR;
1066             }
1067 
1068         } else if (rc != pcmk_ok) {
1069             level = LOG_WARNING;
1070         }
1071 
1072         do_crm_log(level,
1073                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
1074                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
1075                    originator ? originator : "local", client_name, call_id,
1076                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION_ADMIN) : "0",
1077                    the_cib ? crm_element_value(the_cib, XML_ATTR_GENERATION) : "0",
1078                    the_cib ? crm_element_value(the_cib, XML_ATTR_NUMUPDATES) : "0");
1079 
1080         finished = time(NULL);
1081         if ((finished - now) > 3) {
1082             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
1083             crm_write_blackbox(0, NULL);
1084         }
1085 
1086         if (op_reply == NULL && (needs_reply || local_notify)) {
1087             crm_err("Unexpected NULL reply to message");
1088             crm_log_xml_err(request, "null reply");
1089             needs_reply = FALSE;
1090             local_notify = FALSE;
1091         }
1092     }
1093 
1094     if (is_update && !cib_legacy_mode()) {
1095         crm_trace("Completed pre-sync update from %s/%s/%s%s",
1096                   originator ? originator : "local", client_name, call_id,
1097                   local_notify?" with local notification":"");
1098 
1099     } else if (!needs_reply || stand_alone) {
1100         // This was a non-originating slave update
1101         crm_trace("Completed slave update");
1102 
1103     } else if (cib_legacy_mode() &&
1104                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
1105         gboolean broadcast = FALSE;
1106 
1107         cib_local_bcast_num++;
1108         crm_xml_add_int(request, F_CIB_LOCAL_NOTIFY_ID, cib_local_bcast_num);
1109         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
1110 
1111         if (broadcast && client_id && local_notify && op_reply) {
1112 
1113             /* If we have been asked to sync the reply,
1114              * and a bcast msg has gone out, we queue the local notify
1115              * until we know the bcast message has been received */
1116             local_notify = FALSE;
1117             crm_trace("Queuing local %ssync notification for %s",
1118                       (call_options & cib_sync_call) ? "" : "a-", client_id);
1119 
1120             queue_local_notify(op_reply, client_id,
1121                                pcmk_is_set(call_options, cib_sync_call),
1122                                (cib_client == NULL));
1123             op_reply = NULL;    /* the reply is queued, so don't free here */
1124         }
1125 
1126     } else if (call_options & cib_discard_reply) {
1127         crm_trace("Caller isn't interested in reply");
1128 
1129     } else if (cib_client == NULL) {
1130         if (is_update == FALSE || result_diff == NULL) {
1131             crm_trace("Request not broadcast: R/O call");
1132 
1133         } else if (call_options & cib_inhibit_bcast) {
1134             crm_trace("Request not broadcast: inhibited");
1135 
1136         } else if (rc != pcmk_ok) {
1137             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
1138 
1139         } else {
1140             crm_trace("Directing reply to %s", originator);
1141         }
1142 
1143         send_peer_reply(op_reply, result_diff, originator, FALSE);
1144     }
1145 
1146     if (local_notify && client_id) {
1147         crm_trace("Performing local %ssync notification for %s",
1148                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
1149                   client_id);
1150         if (process == FALSE) {
1151             do_local_notify(request, client_id,
1152                             pcmk_is_set(call_options, cib_sync_call),
1153                             (cib_client == NULL));
1154         } else {
1155             do_local_notify(op_reply, client_id,
1156                             pcmk_is_set(call_options, cib_sync_call),
1157                             (cib_client == NULL));
1158         }
1159     }
1160 
1161     free_xml(op_reply);
1162     free_xml(result_diff);
1163 
1164     return;
1165 }
1166 
1167 static int
1168 cib_process_command(xmlNode * request, xmlNode ** reply, xmlNode ** cib_diff, gboolean privileged)
     /*  */
1169 {
1170     xmlNode *input = NULL;
1171     xmlNode *output = NULL;
1172     xmlNode *result_cib = NULL;
1173     xmlNode *current_cib = NULL;
1174 
1175     int call_type = 0;
1176     int call_options = 0;
1177 
1178     const char *op = NULL;
1179     const char *section = NULL;
1180     const char *call_id = crm_element_value(request, F_CIB_CALLID);
1181 
1182     int rc = pcmk_ok;
1183     int rc2 = pcmk_ok;
1184 
1185     gboolean send_r_notify = FALSE;
1186     gboolean global_update = FALSE;
1187     gboolean config_changed = FALSE;
1188     gboolean manage_counters = TRUE;
1189 
1190     static mainloop_timer_t *digest_timer = NULL;
1191 
1192     CRM_ASSERT(cib_status == pcmk_ok);
1193 
1194     if(digest_timer == NULL) {
1195         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1196     }
1197 
1198     *reply = NULL;
1199     *cib_diff = NULL;
1200     current_cib = the_cib;
1201 
1202     /* Start processing the request... */
1203     op = crm_element_value(request, F_CIB_OPERATION);
1204     crm_element_value_int(request, F_CIB_CALLOPTS, &call_options);
1205     rc = cib_get_operation_id(op, &call_type);
1206 
1207     if (rc == pcmk_ok && privileged == FALSE) {
1208         rc = cib_op_can_run(call_type, call_options, privileged, global_update);
1209     }
1210 
1211     rc2 = cib_op_prepare(call_type, request, &input, &section);
1212     if (rc == pcmk_ok) {
1213         rc = rc2;
1214     }
1215 
1216     if (rc != pcmk_ok) {
1217         crm_trace("Call setup failed: %s", pcmk_strerror(rc));
1218         goto done;
1219 
1220     } else if (cib_op_modifies(call_type) == FALSE) {
1221         rc = cib_perform_op(op, call_options, cib_op_func(call_type), TRUE,
1222                             section, request, input, FALSE, &config_changed,
1223                             current_cib, &result_cib, NULL, &output);
1224 
1225         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
1226         goto done;
1227     }
1228 
1229     /* Handle a valid write action */
1230     global_update = crm_is_true(crm_element_value(request, F_CIB_GLOBAL_UPDATE));
1231     if (global_update) {
1232         /* legacy code */
1233         manage_counters = FALSE;
1234         cib__set_call_options(call_options, "call", cib_force_diff);
1235         crm_trace("Global update detected");
1236 
1237         CRM_CHECK(call_type == 3 || call_type == 4, crm_err("Call type: %d", call_type);
1238                   crm_log_xml_err(request, "bad op"));
1239     }
1240 
1241     if (rc == pcmk_ok) {
1242         ping_modified_since = TRUE;
1243         if (call_options & cib_inhibit_bcast) {
1244             /* skip */
1245             crm_trace("Skipping update: inhibit broadcast");
1246             manage_counters = FALSE;
1247         }
1248 
1249         if (!pcmk_is_set(call_options, cib_dryrun)
1250             && pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
1251             /* Copying large CIBs accounts for a huge percentage of our CIB usage */
1252             cib__set_call_options(call_options, "call", cib_zero_copy);
1253         } else {
1254             cib__clear_call_options(call_options, "call", cib_zero_copy);
1255         }
1256 
1257         /* result_cib must not be modified after cib_perform_op() returns */
1258         rc = cib_perform_op(op, call_options, cib_op_func(call_type), FALSE,
1259                             section, request, input, manage_counters, &config_changed,
1260                             current_cib, &result_cib, cib_diff, &output);
1261 
1262         if (manage_counters == FALSE) {
1263             int format = 1;
1264             /* Legacy code
1265              * If the diff is NULL at this point, it's because nothing changed
1266              */
1267             if (*cib_diff) {
1268                 crm_element_value_int(*cib_diff, "format", &format);
1269             }
1270 
1271             if (format == 1) {
1272                 config_changed = cib_config_changed(NULL, NULL, cib_diff);
1273             }
1274         }
1275 
1276         /* Always write to disk for replace ops,
1277          * this also negates the need to detect ordering changes
1278          */
1279         if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
1280             config_changed = TRUE;
1281         }
1282     }
1283 
1284     if (rc == pcmk_ok && !pcmk_is_set(call_options, cib_dryrun)) {
1285         crm_trace("Activating %s->%s%s%s",
1286                   crm_element_value(current_cib, XML_ATTR_NUMUPDATES),
1287                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES),
1288                   (pcmk_is_set(call_options, cib_zero_copy)? " zero-copy" : ""),
1289                   (config_changed? " changed" : ""));
1290         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1291             rc = activateCibXml(result_cib, config_changed, op);
1292             crm_trace("Activated %s (%d)",
1293                       crm_element_value(current_cib, XML_ATTR_NUMUPDATES), rc);
1294         }
1295 
1296         if (rc == pcmk_ok && cib_internal_config_changed(*cib_diff)) {
1297             cib_read_config(config_hash, result_cib);
1298         }
1299 
1300         if (pcmk__str_eq(CIB_OP_REPLACE, op, pcmk__str_none)) {
1301             if (section == NULL) {
1302                 send_r_notify = TRUE;
1303 
1304             } else if (pcmk__str_eq(section, XML_TAG_CIB, pcmk__str_casei)) {
1305                 send_r_notify = TRUE;
1306 
1307             } else if (pcmk__str_eq(section, XML_CIB_TAG_NODES, pcmk__str_casei)) {
1308                 send_r_notify = TRUE;
1309 
1310             } else if (pcmk__str_eq(section, XML_CIB_TAG_STATUS, pcmk__str_casei)) {
1311                 send_r_notify = TRUE;
1312 
1313             } else if (pcmk__str_eq(section, XML_CIB_TAG_CONFIGURATION, pcmk__str_casei)) {
1314                 send_r_notify = TRUE;
1315             }
1316 
1317         } else if (pcmk__str_eq(CIB_OP_ERASE, op, pcmk__str_none)) {
1318             send_r_notify = TRUE;
1319         }
1320 
1321         mainloop_timer_stop(digest_timer);
1322         mainloop_timer_start(digest_timer);
1323 
1324     } else if (rc == -pcmk_err_schema_validation) {
1325         CRM_ASSERT(!pcmk_is_set(call_options, cib_zero_copy));
1326 
1327         if (output != NULL) {
1328             crm_log_xml_info(output, "cib:output");
1329             free_xml(output);
1330         }
1331 
1332         output = result_cib;
1333 
1334     } else {
1335         crm_trace("Not activating %d %d %s", rc,
1336                   pcmk_is_set(call_options, cib_dryrun),
1337                   crm_element_value(result_cib, XML_ATTR_NUMUPDATES));
1338         if (!pcmk_is_set(call_options, cib_zero_copy)) {
1339             free_xml(result_cib);
1340         }
1341     }
1342 
1343     if ((call_options & (cib_inhibit_notify|cib_dryrun)) == 0) {
1344         const char *client = crm_element_value(request, F_CIB_CLIENTNAME);
1345 
1346         crm_trace("Sending notifications %d",
1347                   pcmk_is_set(call_options, cib_dryrun));
1348         cib_diff_notify(call_options, client, call_id, op, input, rc, *cib_diff);
1349     }
1350 
1351     if (send_r_notify) {
1352         const char *origin = crm_element_value(request, F_ORIG);
1353 
1354         cib_replace_notify(origin, the_cib, rc, *cib_diff);
1355     }
1356 
1357     xml_log_patchset(LOG_TRACE, "cib:diff", *cib_diff);
1358   done:
1359     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
1360         const char *caller = crm_element_value(request, F_CIB_CLIENTID);
1361 
1362         *reply = create_xml_node(NULL, "cib-reply");
1363         crm_xml_add(*reply, F_TYPE, T_CIB);
1364         crm_xml_add(*reply, F_CIB_OPERATION, op);
1365         crm_xml_add(*reply, F_CIB_CALLID, call_id);
1366         crm_xml_add(*reply, F_CIB_CLIENTID, caller);
1367         crm_xml_add_int(*reply, F_CIB_CALLOPTS, call_options);
1368         crm_xml_add_int(*reply, F_CIB_RC, rc);
1369 
1370         if (output != NULL) {
1371             crm_trace("Attaching reply output");
1372             add_message_xml(*reply, F_CIB_CALLDATA, output);
1373         }
1374 
1375         crm_log_xml_explicit(*reply, "cib:reply");
1376     }
1377 
1378     crm_trace("cleanup");
1379 
1380     if (cib_op_modifies(call_type) == FALSE && output != current_cib) {
1381         free_xml(output);
1382         output = NULL;
1383     }
1384 
1385     if (call_type >= 0) {
1386         cib_op_cleanup(call_type, call_options, &input, &output);
1387     }
1388 
1389     crm_trace("done");
1390     return rc;
1391 }
1392 
1393 void
1394 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1395 {
1396     const char *reason = NULL;
1397     const char *originator = crm_element_value(msg, F_ORIG);
1398 
1399     if (cib_legacy_mode() && pcmk__str_eq(originator, cib_our_uname, pcmk__str_null_matches)) {
1400         /* message is from ourselves */
1401         int bcast_id = 0;
1402 
1403         if (!(crm_element_value_int(msg, F_CIB_LOCAL_NOTIFY_ID, &bcast_id))) {
1404             check_local_notify(bcast_id);
1405         }
1406         return;
1407 
1408     } else if (crm_peer_cache == NULL) {
1409         reason = "membership not established";
1410         goto bail;
1411     }
1412 
1413     if (crm_element_value(msg, F_CIB_CLIENTNAME) == NULL) {
1414         crm_xml_add(msg, F_CIB_CLIENTNAME, originator);
1415     }
1416 
1417     /* crm_log_xml_trace("Peer[inbound]", msg); */
1418     cib_process_request(msg, TRUE, NULL);
1419     return;
1420 
1421   bail:
1422     if (reason) {
1423         const char *seq = crm_element_value(msg, F_SEQ);
1424         const char *op = crm_element_value(msg, F_CIB_OPERATION);
1425 
1426         crm_warn("Discarding %s message (%s) from %s: %s", op, seq, originator, reason);
1427     }
1428 }
1429 
1430 static gboolean
1431 cib_force_exit(gpointer data)
     /*  */
1432 {
1433     crm_notice("Forcing exit!");
1434     terminate_cib(__func__, CRM_EX_ERROR);
1435     return FALSE;
1436 }
1437 
1438 static void
1439 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1440 {
1441     pcmk__client_t *a_client = value;
1442 
1443     crm_err("Can't disconnect client %s: Not implemented",
1444             pcmk__client_name(a_client));
1445 }
1446 
1447 void
1448 cib_shutdown(int nsig)
     /*  */
1449 {
1450     struct qb_ipcs_stats srv_stats;
1451 
1452     if (cib_shutdown_flag == FALSE) {
1453         int disconnects = 0;
1454         qb_ipcs_connection_t *c = NULL;
1455 
1456         cib_shutdown_flag = TRUE;
1457 
1458         c = qb_ipcs_connection_first_get(ipcs_rw);
1459         while (c != NULL) {
1460             qb_ipcs_connection_t *last = c;
1461 
1462             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1463 
1464             crm_debug("Disconnecting r/w client %p...", last);
1465             qb_ipcs_disconnect(last);
1466             qb_ipcs_connection_unref(last);
1467             disconnects++;
1468         }
1469 
1470         c = qb_ipcs_connection_first_get(ipcs_ro);
1471         while (c != NULL) {
1472             qb_ipcs_connection_t *last = c;
1473 
1474             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1475 
1476             crm_debug("Disconnecting r/o client %p...", last);
1477             qb_ipcs_disconnect(last);
1478             qb_ipcs_connection_unref(last);
1479             disconnects++;
1480         }
1481 
1482         c = qb_ipcs_connection_first_get(ipcs_shm);
1483         while (c != NULL) {
1484             qb_ipcs_connection_t *last = c;
1485 
1486             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1487 
1488             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1489             qb_ipcs_disconnect(last);
1490             qb_ipcs_connection_unref(last);
1491             disconnects++;
1492         }
1493 
1494         disconnects += pcmk__ipc_client_count();
1495 
1496         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1497         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1498         crm_info("Disconnected %d clients", disconnects);
1499     }
1500 
1501     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1502 
1503     if (pcmk__ipc_client_count() == 0) {
1504         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1505         initiate_exit();
1506 
1507     } else {
1508         crm_info("Waiting on %d clients to disconnect (%d)",
1509                  pcmk__ipc_client_count(), srv_stats.active_connections);
1510     }
1511 }
1512 
1513 void
1514 initiate_exit(void)
     /*  */
1515 {
1516     int active = 0;
1517     xmlNode *leaving = NULL;
1518 
1519     active = crm_active_peers();
1520     if (active < 2) {
1521         terminate_cib(__func__, 0);
1522         return;
1523     }
1524 
1525     crm_info("Sending disconnect notification to %d peers...", active);
1526 
1527     leaving = create_xml_node(NULL, "exit-notification");
1528     crm_xml_add(leaving, F_TYPE, "cib");
1529     crm_xml_add(leaving, F_CIB_OPERATION, "cib_shutdown_req");
1530 
1531     send_cluster_message(NULL, crm_msg_cib, leaving, TRUE);
1532     free_xml(leaving);
1533 
1534     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1535 }
1536 
1537 extern int remote_fd;
1538 extern int remote_tls_fd;
1539 
1540 /*!
1541  * \internal
1542  * \brief Close remote sockets, free the global CIB and quit
1543  *
1544  * \param[in] caller           Name of calling function (for log message)
1545  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1546  */
1547 void
1548 terminate_cib(const char *caller, int fast)
     /*  */
1549 {
1550     crm_info("%s: Exiting%s...", caller,
1551              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1552 
1553     if (remote_fd > 0) {
1554         close(remote_fd);
1555         remote_fd = 0;
1556     }
1557     if (remote_tls_fd > 0) {
1558         close(remote_tls_fd);
1559         remote_tls_fd = 0;
1560     }
1561 
1562     uninitializeCib();
1563 
1564     if (fast > 0) {
1565         /* Quit fast on error */
1566         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1567         crm_exit(fast);
1568 
1569     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1570         /* Quit via returning from the main loop. If fast == -1, we skip the
1571          * disconnect here, and it will be done when the main loop returns
1572          * (this allows the peer status callback to avoid messing with the
1573          * peer caches).
1574          */
1575         if (fast == 0) {
1576             crm_cluster_disconnect(&crm_cluster);
1577         }
1578         g_main_loop_quit(mainloop);
1579 
1580     } else {
1581         /* Quit via clean exit. Even the peer status callback can disconnect
1582          * here, because we're not returning control to the caller. */
1583         crm_cluster_disconnect(&crm_cluster);
1584         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1585         crm_exit(CRM_EX_OK);
1586     }
1587 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
