daemons/based/based

/* */
This source file includes following definitions.
cib_legacy_mode
cib_ipc_accept
cib_ipc_dispatch_rw
cib_ipc_dispatch_ro
cib_ipc_closed
cib_ipc_destroy
create_cib_reply
do_local_notify
cib_common_callback_worker
cib_common_callback
cib_digester_cb
process_ping_reply
local_notify_destroy_callback
check_local_notify
queue_local_notify
parse_local_options_v1
parse_local_options_v2
parse_local_options
parse_peer_options_v1
parse_peer_options_v2
parse_peer_options
forward_request
send_peer_reply
cib_process_request
prepare_input
contains_config_change
cib_process_command
cib_peer_callback
cib_force_exit
disconnect_remote_client
initiate_exit
cib_shutdown
terminate_cib
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <sys/types.h>
  15 #include <unistd.h>
  16 
  17 #include <stdlib.h>
  18 #include <stdint.h>     // uint32_t, uint64_t, UINT64_C()
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <inttypes.h>   // PRIu64
  22 
  23 #include <glib.h>
  24 #include <libxml/tree.h>
  25 
  26 #include <crm/crm.h>
  27 #include <crm/cib.h>
  28 #include <crm/cluster/internal.h>
  29 
  30 #include <crm/common/xml.h>
  31 #include <crm/common/remote_internal.h>
  32 
  33 #include <pacemaker-based.h>
  34 
  35 #define EXIT_ESCALATION_MS 10000
  36 
  37 static unsigned long cib_local_bcast_num = 0;
  38 
  39 typedef struct cib_local_notify_s {
  40     xmlNode *notify_src;
  41     char *client_id;
  42     gboolean from_peer;
  43     gboolean sync_reply;
  44 } cib_local_notify_t;
  45 
  46 int next_client_id = 0;
  47 
  48 gboolean legacy_mode = FALSE;
  49 
  50 qb_ipcs_service_t *ipcs_ro = NULL;
  51 qb_ipcs_service_t *ipcs_rw = NULL;
  52 qb_ipcs_service_t *ipcs_shm = NULL;
  53 
  54 static int cib_process_command(xmlNode *request,
  55                                const cib__operation_t *operation,
  56                                cib__op_fn_t op_function, xmlNode **reply,
  57                                xmlNode **cib_diff, bool privileged);
  58 
  59 static gboolean cib_common_callback(qb_ipcs_connection_t *c, void *data,
  60                                     size_t size, gboolean privileged);
  61 
  62 gboolean
  63 cib_legacy_mode(void)
     /*  */
  64 {
  65     return legacy_mode;
  66 }
  67 
  68 static int32_t
  69 cib_ipc_accept(qb_ipcs_connection_t * c, uid_t uid, gid_t gid)
     /*  */
  70 {
  71     if (cib_shutdown_flag) {
  72         crm_info("Ignoring new IPC client [%d] during shutdown",
  73                  pcmk__client_pid(c));
  74         return -ECONNREFUSED;
  75     }
  76 
  77     if (pcmk__new_client(c, uid, gid) == NULL) {
  78         return -ENOMEM;
  79     }
  80     return 0;
  81 }
  82 
  83 static int32_t
  84 cib_ipc_dispatch_rw(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  85 {
  86     pcmk__client_t *client = pcmk__find_client(c);
  87 
  88     crm_trace("%p message from %s", c, client->id);
  89     return cib_common_callback(c, data, size, TRUE);
  90 }
  91 
  92 static int32_t
  93 cib_ipc_dispatch_ro(qb_ipcs_connection_t * c, void *data, size_t size)
     /*  */
  94 {
  95     pcmk__client_t *client = pcmk__find_client(c);
  96 
  97     crm_trace("%p message from %s", c, client->id);
  98     return cib_common_callback(c, data, size, FALSE);
  99 }
 100 
 101 /* Error code means? */
 102 static int32_t
 103 cib_ipc_closed(qb_ipcs_connection_t * c)
     /*  */
 104 {
 105     pcmk__client_t *client = pcmk__find_client(c);
 106 
 107     if (client == NULL) {
 108         return 0;
 109     }
 110     crm_trace("Connection %p", c);
 111     pcmk__free_client(client);
 112     return 0;
 113 }
 114 
 115 static void
 116 cib_ipc_destroy(qb_ipcs_connection_t * c)
     /*  */
 117 {
 118     crm_trace("Connection %p", c);
 119     cib_ipc_closed(c);
 120     if (cib_shutdown_flag) {
 121         cib_shutdown(0);
 122     }
 123 }
 124 
 125 struct qb_ipcs_service_handlers ipc_ro_callbacks = {
 126     .connection_accept = cib_ipc_accept,
 127     .connection_created = NULL,
 128     .msg_process = cib_ipc_dispatch_ro,
 129     .connection_closed = cib_ipc_closed,
 130     .connection_destroyed = cib_ipc_destroy
 131 };
 132 
 133 struct qb_ipcs_service_handlers ipc_rw_callbacks = {
 134     .connection_accept = cib_ipc_accept,
 135     .connection_created = NULL,
 136     .msg_process = cib_ipc_dispatch_rw,
 137     .connection_closed = cib_ipc_closed,
 138     .connection_destroyed = cib_ipc_destroy
 139 };
 140 
 141 /*!
 142  * \internal
 143  * \brief Create reply XML for a CIB request
 144  *
 145  * \param[in] op            CIB operation type
 146  * \param[in] call_id       CIB call ID
 147  * \param[in] client_id     CIB client ID
 148  * \param[in] call_options  Group of <tt>enum cib_call_options</tt> flags
 149  * \param[in] rc            Request return code
 150  * \param[in] call_data     Request output data
 151  *
 152  * \return Reply XML
 153  *
 154  * \note The caller is responsible for freeing the return value using
 155  *       \p free_xml().
 156  */
 157 static xmlNode *
 158 create_cib_reply(const char *op, const char *call_id, const char *client_id,
     /*  */
 159                  uint32_t call_options, int rc, xmlNode *call_data)
 160 {
 161     xmlNode *reply = pcmk__xe_create(NULL, PCMK__XE_CIB_REPLY);
 162 
 163     crm_xml_add(reply, PCMK__XA_T, PCMK__VALUE_CIB);
 164     crm_xml_add(reply, PCMK__XA_CIB_OP, op);
 165     crm_xml_add(reply, PCMK__XA_CIB_CALLID, call_id);
 166     crm_xml_add(reply, PCMK__XA_CIB_CLIENTID, client_id);
 167     crm_xml_add_int(reply, PCMK__XA_CIB_CALLOPT, call_options);
 168     crm_xml_add_int(reply, PCMK__XA_CIB_RC, rc);
 169 
 170     if (call_data != NULL) {
 171         xmlNode *wrapper = pcmk__xe_create(reply, PCMK__XE_CIB_CALLDATA);
 172 
 173         crm_trace("Attaching reply output");
 174         pcmk__xml_copy(wrapper, call_data);
 175     }
 176 
 177     crm_log_xml_explicit(reply, "cib:reply");
 178     return reply;
 179 }
 180 
 181 static void
 182 do_local_notify(const xmlNode *notify_src, const char *client_id,
     /*  */
 183                 bool sync_reply, bool from_peer)
 184 {
 185     int msg_id = 0;
 186     int rc = pcmk_rc_ok;
 187     pcmk__client_t *client_obj = NULL;
 188     uint32_t flags = crm_ipc_server_event;
 189 
 190     CRM_CHECK((notify_src != NULL) && (client_id != NULL), return);
 191 
 192     crm_element_value_int(notify_src, PCMK__XA_CIB_CALLID, &msg_id);
 193 
 194     client_obj = pcmk__find_client_by_id(client_id);
 195     if (client_obj == NULL) {
 196         crm_debug("Could not notify client %s%s %s of call %d result: "
 197                   "client no longer exists", client_id,
 198                   (from_peer? " (originator of delegated request)" : ""),
 199                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 200         return;
 201     }
 202 
 203     if (sync_reply) {
 204         flags = crm_ipc_flags_none;
 205         if (client_obj->ipcs != NULL) {
 206             msg_id = client_obj->request_id;
 207             client_obj->request_id = 0;
 208         }
 209     }
 210 
 211     switch (PCMK__CLIENT_TYPE(client_obj)) {
 212         case pcmk__client_ipc:
 213             rc = pcmk__ipc_send_xml(client_obj, msg_id, notify_src, flags);
 214             break;
 215 #ifdef HAVE_GNUTLS_GNUTLS_H
 216         case pcmk__client_tls:
 217 #endif
 218         case pcmk__client_tcp:
 219             rc = pcmk__remote_send_xml(client_obj->remote, notify_src);
 220             break;
 221         default:
 222             rc = EPROTONOSUPPORT;
 223             break;
 224     }
 225     if (rc == pcmk_rc_ok) {
 226         crm_trace("Notified %s client %s%s %s of call %d result",
 227                   pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 228                   pcmk__client_name(client_obj),
 229                   (from_peer? " (originator of delegated request)" : ""),
 230                   (sync_reply? "synchronously" : "asynchronously"), msg_id);
 231     } else {
 232         crm_warn("Could not notify %s client %s%s %s of call %d result: %s",
 233                  pcmk__client_type_str(PCMK__CLIENT_TYPE(client_obj)),
 234                  pcmk__client_name(client_obj),
 235                  (from_peer? " (originator of delegated request)" : ""),
 236                  (sync_reply? "synchronously" : "asynchronously"), msg_id,
 237                  pcmk_rc_str(rc));
 238     }
 239 }
 240 
 241 void
 242 cib_common_callback_worker(uint32_t id, uint32_t flags, xmlNode * op_request,
     /*  */
 243                            pcmk__client_t *cib_client, gboolean privileged)
 244 {
 245     const char *op = crm_element_value(op_request, PCMK__XA_CIB_OP);
 246     uint32_t call_options = cib_none;
 247     int rc = pcmk_rc_ok;
 248 
 249     rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 250                             cib_none);
 251     if (rc != pcmk_rc_ok) {
 252         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
 253     }
 254 
 255     /* Requests with cib_transaction set should not be sent to based directly
 256      * (outside of a commit-transaction request)
 257      */
 258     if (pcmk_is_set(call_options, cib_transaction)) {
 259         return;
 260     }
 261 
 262     if (pcmk__str_eq(op, CRM_OP_REGISTER, pcmk__str_none)) {
 263         if (flags & crm_ipc_client_response) {
 264             xmlNode *ack = pcmk__xe_create(NULL, __func__);
 265 
 266             crm_xml_add(ack, PCMK__XA_CIB_OP, CRM_OP_REGISTER);
 267             crm_xml_add(ack, PCMK__XA_CIB_CLIENTID, cib_client->id);
 268             pcmk__ipc_send_xml(cib_client, id, ack, flags);
 269             cib_client->request_id = 0;
 270             free_xml(ack);
 271         }
 272         return;
 273 
 274     } else if (pcmk__str_eq(op, PCMK__VALUE_CIB_NOTIFY, pcmk__str_none)) {
 275         /* Update the notify filters for this client */
 276         int on_off = 0;
 277         crm_exit_t status = CRM_EX_OK;
 278         uint64_t bit = UINT64_C(0);
 279         const char *type = crm_element_value(op_request,
 280                                              PCMK__XA_CIB_NOTIFY_TYPE);
 281 
 282         crm_element_value_int(op_request, PCMK__XA_CIB_NOTIFY_ACTIVATE,
 283                               &on_off);
 284 
 285         crm_debug("Setting %s callbacks %s for client %s",
 286                   type, (on_off? "on" : "off"), pcmk__client_name(cib_client));
 287 
 288         if (pcmk__str_eq(type, PCMK__VALUE_CIB_POST_NOTIFY, pcmk__str_none)) {
 289             bit = cib_notify_post;
 290 
 291         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_PRE_NOTIFY,
 292                                 pcmk__str_none)) {
 293             bit = cib_notify_pre;
 294 
 295         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_UPDATE_CONFIRMATION,
 296                                 pcmk__str_none)) {
 297             bit = cib_notify_confirm;
 298 
 299         } else if (pcmk__str_eq(type, PCMK__VALUE_CIB_DIFF_NOTIFY,
 300                                 pcmk__str_none)) {
 301             bit = cib_notify_diff;
 302 
 303         } else {
 304             status = CRM_EX_INVALID_PARAM;
 305         }
 306 
 307         if (bit != 0) {
 308             if (on_off) {
 309                 pcmk__set_client_flags(cib_client, bit);
 310             } else {
 311                 pcmk__clear_client_flags(cib_client, bit);
 312             }
 313         }
 314 
 315         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_ACK, NULL, status);
 316         return;
 317     }
 318 
 319     cib_process_request(op_request, privileged, cib_client);
 320 }
 321 
 322 int32_t
 323 cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean privileged)
     /*  */
 324 {
 325     uint32_t id = 0;
 326     uint32_t flags = 0;
 327     uint32_t call_options = cib_none;
 328     pcmk__client_t *cib_client = pcmk__find_client(c);
 329     xmlNode *op_request = pcmk__client_data2xml(cib_client, data, &id, &flags);
 330 
 331     if (op_request) {
 332         int rc = pcmk_rc_ok;
 333 
 334         rc = pcmk__xe_get_flags(op_request, PCMK__XA_CIB_CALLOPT, &call_options,
 335                                 cib_none);
 336         if (rc != pcmk_rc_ok) {
 337             crm_warn("Couldn't parse options from request: %s",
 338                      pcmk_rc_str(rc));
 339         }
 340     }
 341 
 342     if (op_request == NULL) {
 343         crm_trace("Invalid message from %p", c);
 344         pcmk__ipc_send_ack(cib_client, id, flags, PCMK__XE_NACK, NULL,
 345                            CRM_EX_PROTOCOL);
 346         return 0;
 347 
 348     } else if(cib_client == NULL) {
 349         crm_trace("Invalid client %p", c);
 350         return 0;
 351     }
 352 
 353     if (pcmk_is_set(call_options, cib_sync_call)) {
 354         CRM_LOG_ASSERT(flags & crm_ipc_client_response);
 355         CRM_LOG_ASSERT(cib_client->request_id == 0);    /* This means the client has two synchronous events in-flight */
 356         cib_client->request_id = id;    /* Reply only to the last one */
 357     }
 358 
 359     if (cib_client->name == NULL) {
 360         const char *value = crm_element_value(op_request,
 361                                               PCMK__XA_CIB_CLIENTNAME);
 362 
 363         if (value == NULL) {
 364             cib_client->name = pcmk__itoa(cib_client->pid);
 365         } else {
 366             cib_client->name = pcmk__str_copy(value);
 367             if (crm_is_daemon_name(value)) {
 368                 pcmk__set_client_flags(cib_client, cib_is_daemon);
 369             }
 370         }
 371     }
 372 
 373     /* Allow cluster daemons more leeway before being evicted */
 374     if (pcmk_is_set(cib_client->flags, cib_is_daemon)) {
 375         const char *qmax = cib_config_lookup(PCMK_OPT_CLUSTER_IPC_LIMIT);
 376 
 377         pcmk__set_client_queue_max(cib_client, qmax);
 378     }
 379 
 380     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTID, cib_client->id);
 381     crm_xml_add(op_request, PCMK__XA_CIB_CLIENTNAME, cib_client->name);
 382 
 383     CRM_LOG_ASSERT(cib_client->user != NULL);
 384     pcmk__update_acl_user(op_request, PCMK__XA_CIB_USER, cib_client->user);
 385 
 386     cib_common_callback_worker(id, flags, op_request, cib_client, privileged);
 387     free_xml(op_request);
 388 
 389     return 0;
 390 }
 391 
 392 static uint64_t ping_seq = 0;
 393 static char *ping_digest = NULL;
 394 static bool ping_modified_since = FALSE;
 395 
 396 static gboolean
 397 cib_digester_cb(gpointer data)
     /*  */
 398 {
 399     if (based_is_primary) {
 400         char buffer[32];
 401         xmlNode *ping = pcmk__xe_create(NULL, PCMK__XE_PING);
 402 
 403         ping_seq++;
 404         free(ping_digest);
 405         ping_digest = NULL;
 406         ping_modified_since = FALSE;
 407         snprintf(buffer, 32, "%" PRIu64, ping_seq);
 408         crm_trace("Requesting peer digests (%s)", buffer);
 409 
 410         crm_xml_add(ping, PCMK__XA_T, PCMK__VALUE_CIB);
 411         crm_xml_add(ping, PCMK__XA_CIB_OP, CRM_OP_PING);
 412         crm_xml_add(ping, PCMK__XA_CIB_PING_ID, buffer);
 413 
 414         crm_xml_add(ping, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
 415         pcmk__cluster_send_message(NULL, crm_msg_cib, ping);
 416 
 417         free_xml(ping);
 418     }
 419     return FALSE;
 420 }
 421 
 422 static void
 423 process_ping_reply(xmlNode *reply) 
     /*  */
 424 {
 425     uint64_t seq = 0;
 426     const char *host = crm_element_value(reply, PCMK__XA_SRC);
 427 
 428     xmlNode *wrapper = pcmk__xe_first_child(reply, PCMK__XE_CIB_CALLDATA, NULL,
 429                                             NULL);
 430     xmlNode *pong = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 431 
 432     const char *seq_s = crm_element_value(pong, PCMK__XA_CIB_PING_ID);
 433     const char *digest = crm_element_value(pong, PCMK__XA_DIGEST);
 434 
 435     if (seq_s == NULL) {
 436         crm_debug("Ignoring ping reply with no " PCMK__XA_CIB_PING_ID);
 437         return;
 438 
 439     } else {
 440         long long seq_ll;
 441         int rc = pcmk__scan_ll(seq_s, &seq_ll, 0LL);
 442 
 443         if (rc != pcmk_rc_ok) {
 444             crm_debug("Ignoring ping reply with invalid " PCMK__XA_CIB_PING_ID
 445                       " '%s': %s", seq_s, pcmk_rc_str(rc));
 446             return;
 447         }
 448         seq = (uint64_t) seq_ll;
 449     }
 450 
 451     if(digest == NULL) {
 452         crm_trace("Ignoring ping reply %s from %s with no digest", seq_s, host);
 453 
 454     } else if(seq != ping_seq) {
 455         crm_trace("Ignoring out of sequence ping reply %s from %s", seq_s, host);
 456 
 457     } else if(ping_modified_since) {
 458         crm_trace("Ignoring ping reply %s from %s: cib updated since", seq_s, host);
 459 
 460     } else {
 461         const char *version = crm_element_value(pong, PCMK_XA_CRM_FEATURE_SET);
 462 
 463         if(ping_digest == NULL) {
 464             crm_trace("Calculating new digest");
 465             ping_digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, version);
 466         }
 467 
 468         crm_trace("Processing ping reply %s from %s (%s)", seq_s, host, digest);
 469         if (!pcmk__str_eq(ping_digest, digest, pcmk__str_casei)) {
 470             xmlNode *wrapper = pcmk__xe_first_child(pong, PCMK__XE_CIB_CALLDATA,
 471                                                     NULL, NULL);
 472             xmlNode *remote_cib = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 473 
 474             const char *admin_epoch_s = NULL;
 475             const char *epoch_s = NULL;
 476             const char *num_updates_s = NULL;
 477 
 478             if (remote_cib != NULL) {
 479                 admin_epoch_s = crm_element_value(remote_cib,
 480                                                   PCMK_XA_ADMIN_EPOCH);
 481                 epoch_s = crm_element_value(remote_cib, PCMK_XA_EPOCH);
 482                 num_updates_s = crm_element_value(remote_cib,
 483                                                   PCMK_XA_NUM_UPDATES);
 484             }
 485 
 486             crm_notice("Local CIB %s.%s.%s.%s differs from %s: %s.%s.%s.%s %p",
 487                        crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH),
 488                        crm_element_value(the_cib, PCMK_XA_EPOCH),
 489                        crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
 490                        ping_digest, host,
 491                        pcmk__s(admin_epoch_s, "_"),
 492                        pcmk__s(epoch_s, "_"),
 493                        pcmk__s(num_updates_s, "_"),
 494                        digest, remote_cib);
 495 
 496             if(remote_cib && remote_cib->children) {
 497                 // Additional debug
 498                 xml_calculate_changes(the_cib, remote_cib);
 499                 pcmk__log_xml_changes(LOG_INFO, remote_cib);
 500                 crm_trace("End of differences");
 501             }
 502 
 503             free_xml(remote_cib);
 504             sync_our_cib(reply, FALSE);
 505         }
 506     }
 507 }
 508 
 509 static void
 510 local_notify_destroy_callback(gpointer data)
     /*  */
 511 {
 512     cib_local_notify_t *notify = data;
 513 
 514     free_xml(notify->notify_src);
 515     free(notify->client_id);
 516     free(notify);
 517 }
 518 
 519 static void
 520 check_local_notify(int bcast_id)
     /*  */
 521 {
 522     const cib_local_notify_t *notify = NULL;
 523 
 524     if (!local_notify_queue) {
 525         return;
 526     }
 527 
 528     notify = pcmk__intkey_table_lookup(local_notify_queue, bcast_id);
 529 
 530     if (notify) {
 531         do_local_notify(notify->notify_src, notify->client_id, notify->sync_reply,
 532                         notify->from_peer);
 533         pcmk__intkey_table_remove(local_notify_queue, bcast_id);
 534     }
 535 }
 536 
 537 static void
 538 queue_local_notify(xmlNode * notify_src, const char *client_id, gboolean sync_reply,
     /*  */
 539                    gboolean from_peer)
 540 {
 541     cib_local_notify_t *notify = pcmk__assert_alloc(1,
 542                                                     sizeof(cib_local_notify_t));
 543 
 544     notify->notify_src = notify_src;
 545     notify->client_id = pcmk__str_copy(client_id);
 546     notify->sync_reply = sync_reply;
 547     notify->from_peer = from_peer;
 548 
 549     if (!local_notify_queue) {
 550         local_notify_queue = pcmk__intkey_table(local_notify_destroy_callback);
 551     }
 552     pcmk__intkey_table_insert(local_notify_queue, cib_local_bcast_num, notify);
 553     // cppcheck doesn't know notify will get freed when hash table is destroyed
 554     // cppcheck-suppress memleak
 555 }
 556 
 557 static void
 558 parse_local_options_v1(const pcmk__client_t *cib_client,
     /*  */
 559                        const cib__operation_t *operation, int call_options,
 560                        const char *host, const char *op, gboolean *local_notify,
 561                        gboolean *needs_reply, gboolean *process,
 562                        gboolean *needs_forward)
 563 {
 564     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 565         && !pcmk_is_set(call_options, cib_inhibit_bcast)) {
 566         /* we need to send an update anyway */
 567         *needs_reply = TRUE;
 568     } else {
 569         *needs_reply = FALSE;
 570     }
 571 
 572     if (host == NULL && (call_options & cib_scope_local)) {
 573         crm_trace("Processing locally scoped %s op from client %s",
 574                   op, pcmk__client_name(cib_client));
 575         *local_notify = TRUE;
 576 
 577     } else if ((host == NULL) && based_is_primary) {
 578         crm_trace("Processing %s op locally from client %s as primary",
 579                   op, pcmk__client_name(cib_client));
 580         *local_notify = TRUE;
 581 
 582     } else if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 583         crm_trace("Processing locally addressed %s op from client %s",
 584                   op, pcmk__client_name(cib_client));
 585         *local_notify = TRUE;
 586 
 587     } else if (stand_alone) {
 588         *needs_forward = FALSE;
 589         *local_notify = TRUE;
 590         *process = TRUE;
 591 
 592     } else {
 593         crm_trace("%s op from %s needs to be forwarded to client %s",
 594                   op, pcmk__client_name(cib_client),
 595                   pcmk__s(host, "the primary instance"));
 596         *needs_forward = TRUE;
 597         *process = FALSE;
 598     }
 599 }
 600 
 601 static void
 602 parse_local_options_v2(const pcmk__client_t *cib_client,
     /*  */
 603                        const cib__operation_t *operation, int call_options,
 604                        const char *host, const char *op, gboolean *local_notify,
 605                        gboolean *needs_reply, gboolean *process,
 606                        gboolean *needs_forward)
 607 {
 608     // Process locally and notify local client
 609     *process = TRUE;
 610     *needs_reply = FALSE;
 611     *local_notify = TRUE;
 612     *needs_forward = FALSE;
 613 
 614     if (pcmk_is_set(operation->flags, cib__op_attr_local)) {
 615         /* Always process locally if cib__op_attr_local is set.
 616          *
 617          * @COMPAT: Currently host is ignored. At a compatibility break, throw
 618          * an error (from cib_process_request() or earlier) if host is not NULL or
 619          * OUR_NODENAME.
 620          */
 621         crm_trace("Processing always-local %s op from client %s",
 622                   op, pcmk__client_name(cib_client));
 623 
 624         if (!pcmk__str_eq(host, OUR_NODENAME,
 625                           pcmk__str_casei|pcmk__str_null_matches)) {
 626 
 627             crm_warn("Operation '%s' is always local but its target host is "
 628                      "set to '%s'",
 629                      op, host);
 630         }
 631         return;
 632     }
 633 
 634     if (pcmk_is_set(operation->flags, cib__op_attr_modifies)
 635         || !pcmk__str_eq(host, OUR_NODENAME,
 636                          pcmk__str_casei|pcmk__str_null_matches)) {
 637 
 638         // Forward modifying and non-local requests via cluster
 639         *process = FALSE;
 640         *needs_reply = FALSE;
 641         *local_notify = FALSE;
 642         *needs_forward = TRUE;
 643 
 644         crm_trace("%s op from %s needs to be forwarded to %s",
 645                   op, pcmk__client_name(cib_client),
 646                   pcmk__s(host, "all nodes"));
 647         return;
 648     }
 649 
 650     if (stand_alone) {
 651         crm_trace("Processing %s op from client %s (stand-alone)",
 652                   op, pcmk__client_name(cib_client));
 653 
 654     } else {
 655         crm_trace("Processing %saddressed %s op from client %s",
 656                   ((host != NULL)? "locally " : "un"),
 657                   op, pcmk__client_name(cib_client));
 658     }
 659 }
 660 
 661 static void
 662 parse_local_options(const pcmk__client_t *cib_client,
     /*  */
 663                     const cib__operation_t *operation, int call_options,
 664                     const char *host, const char *op, gboolean *local_notify,
 665                     gboolean *needs_reply, gboolean *process,
 666                     gboolean *needs_forward)
 667 {
 668     if(cib_legacy_mode()) {
 669         parse_local_options_v1(cib_client, operation, call_options, host,
 670                                op, local_notify, needs_reply, process,
 671                                needs_forward);
 672     } else {
 673         parse_local_options_v2(cib_client, operation, call_options, host,
 674                                op, local_notify, needs_reply, process,
 675                                needs_forward);
 676     }
 677 }
 678 
 679 static gboolean
 680 parse_peer_options_v1(const cib__operation_t *operation, xmlNode *request,
     /*  */
 681                       gboolean *local_notify, gboolean *needs_reply,
 682                       gboolean *process)
 683 {
 684     const char *op = NULL;
 685     const char *host = NULL;
 686     const char *delegated = NULL;
 687     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 688     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 689 
 690     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 691 
 692     if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 693         *needs_reply = FALSE;
 694         if (is_reply) {
 695             *local_notify = TRUE;
 696             crm_trace("Processing global/peer update from %s"
 697                       " that originated from us", originator);
 698         } else {
 699             crm_trace("Processing global/peer update from %s", originator);
 700         }
 701         return TRUE;
 702     }
 703 
 704     op = crm_element_value(request, PCMK__XA_CIB_OP);
 705     crm_trace("Processing legacy %s request sent by %s", op, originator);
 706 
 707     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 708         /* Always process these */
 709         *local_notify = FALSE;
 710         if (reply_to == NULL || is_reply) {
 711             *process = TRUE;
 712         }
 713         if (is_reply) {
 714             *needs_reply = FALSE;
 715         }
 716         return *process;
 717     }
 718 
 719     if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 720         process_ping_reply(request);
 721         return FALSE;
 722     }
 723 
 724     if (is_reply) {
 725         crm_trace("Forward reply sent from %s to local clients", originator);
 726         *process = FALSE;
 727         *needs_reply = FALSE;
 728         *local_notify = TRUE;
 729         return TRUE;
 730     }
 731 
 732     host = crm_element_value(request, PCMK__XA_CIB_HOST);
 733     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 734         crm_trace("Processing %s request sent to us from %s", op, originator);
 735         return TRUE;
 736 
 737     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 738         crm_trace("Processing %s request sent to %s by %s", op, host?host:"everyone", originator);
 739         *needs_reply = TRUE;
 740         return TRUE;
 741 
 742     } else if ((host == NULL) && based_is_primary) {
 743         crm_trace("Processing %s request sent to primary instance from %s",
 744                   op, originator);
 745         return TRUE;
 746     }
 747 
 748     delegated = crm_element_value(request, PCMK__XA_CIB_DELEGATED_FROM);
 749     if (delegated != NULL) {
 750         crm_trace("Ignoring message for primary instance");
 751 
 752     } else if (host != NULL) {
 753         /* this is for a specific instance and we're not it */
 754         crm_trace("Ignoring msg for instance on %s", host);
 755 
 756     } else if ((reply_to == NULL) && !based_is_primary) {
 757         // This is for the primary instance, and we're not it
 758         crm_trace("Ignoring reply for primary instance");
 759 
 760     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 761         if (reply_to != NULL) {
 762             crm_debug("Processing %s from %s", op, originator);
 763             *needs_reply = FALSE;
 764 
 765         } else {
 766             crm_debug("Processing %s reply from %s", op, originator);
 767         }
 768         return TRUE;
 769 
 770     } else {
 771         crm_err("Nothing for us to do?");
 772         crm_log_xml_err(request, "Peer[inbound]");
 773     }
 774 
 775     return FALSE;
 776 }
 777 
 778 static gboolean
 779 parse_peer_options_v2(const cib__operation_t *operation, xmlNode *request,
     /*  */
 780                       gboolean *local_notify, gboolean *needs_reply,
 781                       gboolean *process)
 782 {
 783     const char *host = NULL;
 784     const char *delegated = crm_element_value(request,
 785                                               PCMK__XA_CIB_DELEGATED_FROM);
 786     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 787     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 788     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
 789 
 790     gboolean is_reply = pcmk__str_eq(reply_to, OUR_NODENAME, pcmk__str_casei);
 791 
 792     if (originator == NULL) { // Shouldn't be possible
 793         originator = "peer";
 794     }
 795 
 796     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_REPLACE, pcmk__str_none)) {
 797         // sync_our_cib() sets PCMK__XA_CIB_ISREPLYTO
 798         if (reply_to) {
 799             delegated = reply_to;
 800         }
 801         goto skip_is_reply;
 802 
 803     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SYNC_TO_ALL,
 804                             pcmk__str_none)) {
 805         // Nothing to do
 806 
 807     } else if (is_reply && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 808         process_ping_reply(request);
 809         return FALSE;
 810 
 811     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_UPGRADE, pcmk__str_none)) {
 812         /* Only the DC (node with the oldest software) should process
 813          * this operation if PCMK__XA_CIB_SCHEMA_MAX is unset.
 814          *
 815          * If the DC is happy it will then send out another
 816          * PCMK__CIB_REQUEST_UPGRADE which will tell all nodes to do the actual
 817          * upgrade.
 818          *
 819          * Except this time PCMK__XA_CIB_SCHEMA_MAX will be set which puts a
 820          * limit on how far newer nodes will go
 821          */
 822         const char *max = crm_element_value(request, PCMK__XA_CIB_SCHEMA_MAX);
 823         const char *upgrade_rc = crm_element_value(request,
 824                                                    PCMK__XA_CIB_UPGRADE_RC);
 825 
 826         crm_trace("Parsing upgrade %s for %s with max=%s and upgrade_rc=%s",
 827                   (is_reply? "reply" : "request"),
 828                   (based_is_primary? "primary" : "secondary"),
 829                   pcmk__s(max, "none"), pcmk__s(upgrade_rc, "none"));
 830 
 831         if (upgrade_rc != NULL) {
 832             // Our upgrade request was rejected by DC, notify clients of result
 833             crm_xml_add(request, PCMK__XA_CIB_RC, upgrade_rc);
 834 
 835         } else if ((max == NULL) && based_is_primary) {
 836             /* We are the DC, check if this upgrade is allowed */
 837             goto skip_is_reply;
 838 
 839         } else if(max) {
 840             /* Ok, go ahead and upgrade to 'max' */
 841             goto skip_is_reply;
 842 
 843         } else {
 844             // Ignore broadcast client requests when we're not primary
 845             return FALSE;
 846         }
 847 
 848     } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
 849         crm_info("Detected legacy %s global update from %s", op, originator);
 850         send_sync_request(NULL);
 851         legacy_mode = TRUE;
 852         return FALSE;
 853 
 854     } else if (is_reply
 855                && pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
 856         crm_trace("Ignoring legacy %s reply sent from %s to local clients", op, originator);
 857         return FALSE;
 858 
 859     } else if (pcmk__str_eq(op, PCMK__CIB_REQUEST_SHUTDOWN, pcmk__str_none)) {
 860         *local_notify = FALSE;
 861         if (reply_to == NULL) {
 862             *process = TRUE;
 863         } else { // Not possible?
 864             crm_debug("Ignoring shutdown request from %s because reply_to=%s",
 865                       originator, reply_to);
 866         }
 867         return *process;
 868     }
 869 
 870     if (is_reply) {
 871         crm_trace("Will notify local clients for %s reply from %s",
 872                   op, originator);
 873         *process = FALSE;
 874         *needs_reply = FALSE;
 875         *local_notify = TRUE;
 876         return TRUE;
 877     }
 878 
 879   skip_is_reply:
 880     *process = TRUE;
 881     *needs_reply = FALSE;
 882 
 883     *local_notify = pcmk__str_eq(delegated, OUR_NODENAME, pcmk__str_casei);
 884 
 885     host = crm_element_value(request, PCMK__XA_CIB_HOST);
 886     if (pcmk__str_eq(host, OUR_NODENAME, pcmk__str_casei)) {
 887         crm_trace("Processing %s request sent to us from %s", op, originator);
 888         *needs_reply = TRUE;
 889         return TRUE;
 890 
 891     } else if (host != NULL) {
 892         crm_trace("Ignoring %s request intended for CIB manager on %s",
 893                   op, host);
 894         return FALSE;
 895 
 896     } else if(is_reply == FALSE && pcmk__str_eq(op, CRM_OP_PING, pcmk__str_casei)) {
 897         *needs_reply = TRUE;
 898     }
 899 
 900     crm_trace("Processing %s request broadcast by %s call %s on %s "
 901               "(local clients will%s be notified)", op,
 902               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CLIENTNAME),
 903                       "client"),
 904               pcmk__s(crm_element_value(request, PCMK__XA_CIB_CALLID),
 905                       "without ID"),
 906               originator, (*local_notify? "" : "not"));
 907     return TRUE;
 908 }
 909 
 910 static gboolean
 911 parse_peer_options(const cib__operation_t *operation, xmlNode *request,
     /*  */
 912                    gboolean *local_notify, gboolean *needs_reply,
 913                    gboolean *process)
 914 {
 915     /* TODO: What happens when an update comes in after node A
 916      * requests the CIB from node B, but before it gets the reply (and
 917      * sends out the replace operation)
 918      */
 919     if(cib_legacy_mode()) {
 920         return parse_peer_options_v1(operation, request, local_notify,
 921                                      needs_reply, process);
 922     } else {
 923         return parse_peer_options_v2(operation, request, local_notify,
 924                                      needs_reply, process);
 925     }
 926 }
 927 
 928 /*!
 929  * \internal
 930  * \brief Forward a CIB request to the appropriate target host(s)
 931  *
 932  * \param[in] request  CIB request to forward
 933  */
 934 static void
 935 forward_request(xmlNode *request)
     /*  */
 936 {
 937     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
 938     const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
 939     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
 940     const char *originator = crm_element_value(request, PCMK__XA_SRC);
 941     const char *client_name = crm_element_value(request,
 942                                                 PCMK__XA_CIB_CLIENTNAME);
 943     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
 944     crm_node_t *peer = NULL;
 945 
 946     int log_level = LOG_INFO;
 947 
 948     if (pcmk__str_eq(op, PCMK__CIB_REQUEST_NOOP, pcmk__str_none)) {
 949         log_level = LOG_DEBUG;
 950     }
 951 
 952     do_crm_log(log_level,
 953                "Forwarding %s operation for section %s to %s (origin=%s/%s/%s)",
 954                pcmk__s(op, "invalid"),
 955                pcmk__s(section, "all"),
 956                pcmk__s(host, (cib_legacy_mode()? "primary" : "all")),
 957                pcmk__s(originator, "local"),
 958                pcmk__s(client_name, "unspecified"),
 959                pcmk__s(call_id, "unspecified"));
 960 
 961     crm_xml_add(request, PCMK__XA_CIB_DELEGATED_FROM, OUR_NODENAME);
 962 
 963     if (host != NULL) {
 964         peer = pcmk__get_node(0, host, NULL, pcmk__node_search_cluster_member);
 965     }
 966     pcmk__cluster_send_message(peer, crm_msg_cib, request);
 967 
 968     // Return the request to its original state
 969     pcmk__xe_remove_attr(request, PCMK__XA_CIB_DELEGATED_FROM);
 970 }
 971 
 972 static gboolean
 973 send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
     /*  */
 974 {
 975     pcmk__assert(msg != NULL);
 976 
 977     if (broadcast) {
 978         /* @COMPAT: Legacy code
 979          *
 980          * This successful call modified the CIB, and the change needs to be
 981          * broadcast (sent via cluster to all nodes).
 982          */
 983         int diff_add_updates = 0;
 984         int diff_add_epoch = 0;
 985         int diff_add_admin_epoch = 0;
 986 
 987         int diff_del_updates = 0;
 988         int diff_del_epoch = 0;
 989         int diff_del_admin_epoch = 0;
 990 
 991         const char *digest = NULL;
 992         int format = 1;
 993 
 994         xmlNode *wrapper = NULL;
 995 
 996         CRM_LOG_ASSERT(result_diff != NULL);
 997         digest = crm_element_value(result_diff, PCMK__XA_DIGEST);
 998         crm_element_value_int(result_diff, PCMK_XA_FORMAT, &format);
 999 
1000         cib_diff_version_details(result_diff,
1001                                  &diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
1002                                  &diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
1003 
1004         crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
1005                   diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
1006                   diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
1007 
1008         crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator);
1009         pcmk__xe_set_bool_attr(msg, PCMK__XA_CIB_UPDATE, true);
1010         crm_xml_add(msg, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_APPLY_PATCH);
1011         crm_xml_add(msg, PCMK__XA_CIB_USER, CRM_DAEMON_USER);
1012 
1013         if (format == 1) {
1014             pcmk__assert(digest != NULL);
1015         }
1016 
1017         wrapper = pcmk__xe_create(msg, PCMK__XE_CIB_UPDATE_DIFF);
1018         pcmk__xml_copy(wrapper, result_diff);
1019 
1020         crm_log_xml_explicit(msg, "copy");
1021         return pcmk__cluster_send_message(NULL, crm_msg_cib, msg);
1022 
1023     } else if (originator != NULL) {
1024         /* send reply via HA to originating node */
1025         const crm_node_t *node =
1026             pcmk__get_node(0, originator, NULL,
1027                            pcmk__node_search_cluster_member);
1028 
1029         crm_trace("Sending request result to %s only", originator);
1030         crm_xml_add(msg, PCMK__XA_CIB_ISREPLYTO, originator);
1031         return pcmk__cluster_send_message(node, crm_msg_cib, msg);
1032     }
1033 
1034     return FALSE;
1035 }
1036 
1037 /*!
1038  * \internal
1039  * \brief Handle an IPC or CPG message containing a request
1040  *
1041  * \param[in,out] request        Request XML
1042  * \param[in] privileged         Whether privileged commands may be run
1043  *                               (see cib_server_ops[] definition)
1044  * \param[in] cib_client         IPC client that sent request (or NULL if CPG)
1045  *
1046  * \return Legacy Pacemaker return code
1047  */
1048 int
1049 cib_process_request(xmlNode *request, gboolean privileged,
     /*  */
1050                     const pcmk__client_t *cib_client)
1051 {
1052     // @TODO: Break into multiple smaller functions
1053     uint32_t call_options = cib_none;
1054 
1055     gboolean process = TRUE;        // Whether to process request locally now
1056     gboolean is_update = TRUE;      // Whether request would modify CIB
1057     gboolean needs_reply = TRUE;    // Whether to build a reply
1058     gboolean local_notify = FALSE;  // Whether to notify (local) requester
1059     gboolean needs_forward = FALSE; // Whether to forward request somewhere else
1060 
1061     xmlNode *op_reply = NULL;
1062     xmlNode *result_diff = NULL;
1063 
1064     int rc = pcmk_ok;
1065     const char *op = crm_element_value(request, PCMK__XA_CIB_OP);
1066     const char *originator = crm_element_value(request, PCMK__XA_SRC);
1067     const char *host = crm_element_value(request, PCMK__XA_CIB_HOST);
1068     const char *target = NULL;
1069     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
1070     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
1071     const char *client_name = crm_element_value(request,
1072                                                 PCMK__XA_CIB_CLIENTNAME);
1073     const char *reply_to = crm_element_value(request, PCMK__XA_CIB_ISREPLYTO);
1074 
1075     const cib__operation_t *operation = NULL;
1076     cib__op_fn_t op_function = NULL;
1077 
1078     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
1079                             cib_none);
1080     if (rc != pcmk_rc_ok) {
1081         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
1082     }
1083 
1084     if ((host != NULL) && (*host == '\0')) {
1085         host = NULL;
1086     }
1087 
1088     // @TODO: Improve trace messages. Target is accurate only for legacy mode.
1089     if (host) {
1090         target = host;
1091 
1092     } else if (call_options & cib_scope_local) {
1093         target = "local host";
1094 
1095     } else {
1096         target = "primary";
1097     }
1098 
1099     if (cib_client == NULL) {
1100         crm_trace("Processing peer %s operation from %s/%s on %s intended for %s (reply=%s)",
1101                   op, pcmk__s(client_name, "client"), call_id, originator,
1102                   target, reply_to);
1103     } else {
1104         crm_xml_add(request, PCMK__XA_SRC, OUR_NODENAME);
1105         crm_trace("Processing local %s operation from %s/%s intended for %s",
1106                   op, pcmk__s(client_name, "client"), call_id, target);
1107     }
1108 
1109     rc = cib__get_operation(op, &operation);
1110     rc = pcmk_rc2legacy(rc);
1111     if (rc != pcmk_ok) {
1112         /* TODO: construct error reply? */
1113         crm_err("Pre-processing of command failed: %s", pcmk_strerror(rc));
1114         return rc;
1115     }
1116 
1117     op_function = based_get_op_function(operation);
1118     if (op_function == NULL) {
1119         crm_err("Operation %s not supported by CIB manager", op);
1120         return -EOPNOTSUPP;
1121     }
1122 
1123     if (cib_client != NULL) {
1124         parse_local_options(cib_client, operation, call_options, host, op,
1125                             &local_notify, &needs_reply, &process,
1126                             &needs_forward);
1127 
1128     } else if (!parse_peer_options(operation, request, &local_notify,
1129                                    &needs_reply, &process)) {
1130         return rc;
1131     }
1132 
1133     if (pcmk_is_set(call_options, cib_transaction)) {
1134         /* All requests in a transaction are processed locally against a working
1135          * CIB copy, and we don't notify for individual requests because the
1136          * entire transaction is atomic.
1137          *
1138          * We still call the option parser functions above, for the sake of log
1139          * messages and checking whether we're the target for peer requests.
1140          */
1141         process = TRUE;
1142         needs_reply = FALSE;
1143         local_notify = FALSE;
1144         needs_forward = FALSE;
1145     }
1146 
1147     is_update = pcmk_is_set(operation->flags, cib__op_attr_modifies);
1148 
1149     if (pcmk_is_set(call_options, cib_discard_reply)) {
1150         /* If the request will modify the CIB, and we are in legacy mode, we
1151          * need to build a reply so we can broadcast a diff, even if the
1152          * requester doesn't want one.
1153          */
1154         needs_reply = is_update && cib_legacy_mode();
1155         local_notify = FALSE;
1156         crm_trace("Client is not interested in the reply");
1157     }
1158 
1159     if (needs_forward) {
1160         forward_request(request);
1161         return rc;
1162     }
1163 
1164     if (cib_status != pcmk_ok) {
1165         rc = cib_status;
1166         crm_err("Operation ignored, cluster configuration is invalid."
1167                 " Please repair and restart: %s", pcmk_strerror(cib_status));
1168 
1169         op_reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1170                                     the_cib);
1171 
1172     } else if (process) {
1173         time_t finished = 0;
1174         time_t now = time(NULL);
1175         int level = LOG_INFO;
1176         const char *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
1177         const char *admin_epoch_s = NULL;
1178         const char *epoch_s = NULL;
1179         const char *num_updates_s = NULL;
1180 
1181         rc = cib_process_command(request, operation, op_function, &op_reply,
1182                                  &result_diff, privileged);
1183 
1184         if (!is_update) {
1185             level = LOG_TRACE;
1186 
1187         } else if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
1188             switch (rc) {
1189                 case pcmk_ok:
1190                     level = LOG_INFO;
1191                     break;
1192                 case -pcmk_err_old_data:
1193                 case -pcmk_err_diff_resync:
1194                 case -pcmk_err_diff_failed:
1195                     level = LOG_TRACE;
1196                     break;
1197                 default:
1198                     level = LOG_ERR;
1199             }
1200 
1201         } else if (rc != pcmk_ok) {
1202             level = LOG_WARNING;
1203         }
1204 
1205         if (the_cib != NULL) {
1206             admin_epoch_s = crm_element_value(the_cib, PCMK_XA_ADMIN_EPOCH);
1207             epoch_s = crm_element_value(the_cib, PCMK_XA_EPOCH);
1208             num_updates_s = crm_element_value(the_cib, PCMK_XA_NUM_UPDATES);
1209         }
1210 
1211         do_crm_log(level,
1212                    "Completed %s operation for section %s: %s (rc=%d, origin=%s/%s/%s, version=%s.%s.%s)",
1213                    op, section ? section : "'all'", pcmk_strerror(rc), rc,
1214                    originator ? originator : "local",
1215                    pcmk__s(client_name, "client"), call_id,
1216                    pcmk__s(admin_epoch_s, "0"),
1217                    pcmk__s(epoch_s, "0"),
1218                    pcmk__s(num_updates_s, "0"));
1219 
1220         finished = time(NULL);
1221         if ((finished - now) > 3) {
1222             crm_trace("%s operation took %lds to complete", op, (long)(finished - now));
1223             crm_write_blackbox(0, NULL);
1224         }
1225 
1226         if (op_reply == NULL && (needs_reply || local_notify)) {
1227             crm_err("Unexpected NULL reply to message");
1228             crm_log_xml_err(request, "null reply");
1229             needs_reply = FALSE;
1230             local_notify = FALSE;
1231         }
1232     }
1233 
1234     if (is_update && !cib_legacy_mode()) {
1235         crm_trace("Completed pre-sync update from %s/%s/%s%s",
1236                   originator ? originator : "local",
1237                   pcmk__s(client_name, "client"), call_id,
1238                   local_notify?" with local notification":"");
1239 
1240     } else if (!needs_reply || stand_alone) {
1241         // This was a non-originating secondary update
1242         crm_trace("Completed update as secondary");
1243 
1244     } else if (cib_legacy_mode() &&
1245                rc == pcmk_ok && result_diff != NULL && !(call_options & cib_inhibit_bcast)) {
1246         gboolean broadcast = FALSE;
1247 
1248         cib_local_bcast_num++;
1249         crm_xml_add_int(request, PCMK__XA_CIB_LOCAL_NOTIFY_ID,
1250                         cib_local_bcast_num);
1251         broadcast = send_peer_reply(request, result_diff, originator, TRUE);
1252 
1253         if (broadcast && client_id && local_notify && op_reply) {
1254 
1255             /* If we have been asked to sync the reply,
1256              * and a bcast msg has gone out, we queue the local notify
1257              * until we know the bcast message has been received */
1258             local_notify = FALSE;
1259             crm_trace("Queuing local %ssync notification for %s",
1260                       (call_options & cib_sync_call) ? "" : "a-", client_id);
1261 
1262             queue_local_notify(op_reply, client_id,
1263                                pcmk_is_set(call_options, cib_sync_call),
1264                                (cib_client == NULL));
1265             op_reply = NULL;    /* the reply is queued, so don't free here */
1266         }
1267 
1268     } else if ((cib_client == NULL)
1269                && !pcmk_is_set(call_options, cib_discard_reply)) {
1270 
1271         if (is_update == FALSE || result_diff == NULL) {
1272             crm_trace("Request not broadcast: R/O call");
1273 
1274         } else if (call_options & cib_inhibit_bcast) {
1275             crm_trace("Request not broadcast: inhibited");
1276 
1277         } else if (rc != pcmk_ok) {
1278             crm_trace("Request not broadcast: call failed: %s", pcmk_strerror(rc));
1279 
1280         } else {
1281             crm_trace("Directing reply to %s", originator);
1282         }
1283 
1284         send_peer_reply(op_reply, result_diff, originator, FALSE);
1285     }
1286 
1287     if (local_notify && client_id) {
1288         crm_trace("Performing local %ssync notification for %s",
1289                   (pcmk_is_set(call_options, cib_sync_call)? "" : "a"),
1290                   client_id);
1291         if (process == FALSE) {
1292             do_local_notify(request, client_id,
1293                             pcmk_is_set(call_options, cib_sync_call),
1294                             (cib_client == NULL));
1295         } else {
1296             do_local_notify(op_reply, client_id,
1297                             pcmk_is_set(call_options, cib_sync_call),
1298                             (cib_client == NULL));
1299         }
1300     }
1301 
1302     free_xml(op_reply);
1303     free_xml(result_diff);
1304 
1305     return rc;
1306 }
1307 
1308 /*!
1309  * \internal
1310  * \brief Get a CIB operation's input from the request XML
1311  *
1312  * \param[in]  request  CIB request XML
1313  * \param[in]  type     CIB operation type
1314  * \param[out] section  Where to store CIB section name
1315  *
1316  * \return Input XML for CIB operation
1317  *
1318  * \note If not \c NULL, the return value is a non-const pointer to part of
1319  *       \p request. The caller should not free it directly.
1320  */
1321 static xmlNode *
1322 prepare_input(const xmlNode *request, enum cib__op_type type,
     /*  */
1323               const char **section)
1324 {
1325     xmlNode *wrapper = NULL;
1326     xmlNode *input = NULL;
1327 
1328     *section = NULL;
1329 
1330     switch (type) {
1331         case cib__op_apply_patch:
1332             {
1333                 const char *wrapper_name = PCMK__XE_CIB_CALLDATA;
1334 
1335                 if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
1336                     wrapper_name = PCMK__XE_CIB_UPDATE_DIFF;
1337                 }
1338                 wrapper = pcmk__xe_first_child(request, wrapper_name, NULL,
1339                                                NULL);
1340                 input = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
1341             }
1342             break;
1343 
1344         default:
1345             wrapper = pcmk__xe_first_child(request, PCMK__XE_CIB_CALLDATA, NULL,
1346                                            NULL);
1347             input = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
1348             *section = crm_element_value(request, PCMK__XA_CIB_SECTION);
1349             break;
1350     }
1351 
1352     // Grab the specified section
1353     if ((*section != NULL) && pcmk__xe_is(input, PCMK_XE_CIB)) {
1354         input = pcmk_find_cib_element(input, *section);
1355     }
1356 
1357     return input;
1358 }
1359 
1360 // v1 and v2 patch formats
1361 #define XPATH_CONFIG_CHANGE         \
1362     "//" PCMK_XE_CRM_CONFIG " | "   \
1363     "//" PCMK_XE_CHANGE             \
1364     "[contains(@" PCMK_XA_PATH ",'/" PCMK_XE_CRM_CONFIG "/')]"
1365 
1366 static bool
1367 contains_config_change(xmlNode *diff)
     /*  */
1368 {
1369     bool changed = false;
1370 
1371     if (diff) {
1372         xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
1373 
1374         if (numXpathResults(xpathObj) > 0) {
1375             changed = true;
1376         }
1377         freeXpathObject(xpathObj);
1378     }
1379     return changed;
1380 }
1381 
1382 static int
1383 cib_process_command(xmlNode *request, const cib__operation_t *operation,
     /*  */
1384                     cib__op_fn_t op_function, xmlNode **reply,
1385                     xmlNode **cib_diff, bool privileged)
1386 {
1387     xmlNode *input = NULL;
1388     xmlNode *output = NULL;
1389     xmlNode *result_cib = NULL;
1390 
1391     uint32_t call_options = cib_none;
1392 
1393     const char *op = NULL;
1394     const char *section = NULL;
1395     const char *call_id = crm_element_value(request, PCMK__XA_CIB_CALLID);
1396     const char *client_id = crm_element_value(request, PCMK__XA_CIB_CLIENTID);
1397     const char *client_name = crm_element_value(request,
1398                                                 PCMK__XA_CIB_CLIENTNAME);
1399     const char *originator = crm_element_value(request, PCMK__XA_SRC);
1400 
1401     int rc = pcmk_ok;
1402 
1403     bool config_changed = false;
1404     bool manage_counters = true;
1405 
1406     static mainloop_timer_t *digest_timer = NULL;
1407 
1408     pcmk__assert(cib_status == pcmk_ok);
1409 
1410     if(digest_timer == NULL) {
1411         digest_timer = mainloop_timer_add("digester", 5000, FALSE, cib_digester_cb, NULL);
1412     }
1413 
1414     *reply = NULL;
1415     *cib_diff = NULL;
1416 
1417     /* Start processing the request... */
1418     op = crm_element_value(request, PCMK__XA_CIB_OP);
1419     rc = pcmk__xe_get_flags(request, PCMK__XA_CIB_CALLOPT, &call_options,
1420                             cib_none);
1421     if (rc != pcmk_rc_ok) {
1422         crm_warn("Couldn't parse options from request: %s", pcmk_rc_str(rc));
1423     }
1424 
1425     if (!privileged && pcmk_is_set(operation->flags, cib__op_attr_privileged)) {
1426         rc = -EACCES;
1427         crm_trace("Failed due to lack of privileges: %s", pcmk_strerror(rc));
1428         goto done;
1429     }
1430 
1431     input = prepare_input(request, operation->type, &section);
1432 
1433     if (!pcmk_is_set(operation->flags, cib__op_attr_modifies)) {
1434         rc = cib_perform_op(NULL, op, call_options, op_function, true, section,
1435                             request, input, false, &config_changed, &the_cib,
1436                             &result_cib, NULL, &output);
1437 
1438         CRM_CHECK(result_cib == NULL, free_xml(result_cib));
1439         goto done;
1440     }
1441 
1442     /* @COMPAT: Handle a valid write action (legacy)
1443      *
1444      * @TODO: Re-evaluate whether this is all truly legacy. The cib_force_diff
1445      * portion is. However, PCMK__XA_CIB_UPDATE may be set by a sync operation
1446      * even in non-legacy mode, and manage_counters tells xml_create_patchset()
1447      * whether to update version/epoch info.
1448      */
1449     if (pcmk__xe_attr_is_true(request, PCMK__XA_CIB_UPDATE)) {
1450         manage_counters = false;
1451         cib__set_call_options(call_options, "call", cib_force_diff);
1452         crm_trace("Global update detected");
1453 
1454         CRM_LOG_ASSERT(pcmk__str_any_of(op,
1455                                         PCMK__CIB_REQUEST_APPLY_PATCH,
1456                                         PCMK__CIB_REQUEST_REPLACE,
1457                                         NULL));
1458     }
1459 
1460     ping_modified_since = TRUE;
1461     if (pcmk_is_set(call_options, cib_inhibit_bcast)) {
1462         crm_trace("Skipping update: inhibit broadcast");
1463         manage_counters = false;
1464     }
1465 
1466     // result_cib must not be modified after cib_perform_op() returns
1467     rc = cib_perform_op(NULL, op, call_options, op_function, false, section,
1468                         request, input, manage_counters, &config_changed,
1469                         &the_cib, &result_cib, cib_diff, &output);
1470 
1471     // @COMPAT: Legacy code
1472     if (!manage_counters) {
1473         int format = 1;
1474 
1475         // If the diff is NULL at this point, it's because nothing changed
1476         if (*cib_diff != NULL) {
1477             crm_element_value_int(*cib_diff, PCMK_XA_FORMAT, &format);
1478         }
1479 
1480         if (format == 1) {
1481             config_changed = cib__config_changed_v1(NULL, NULL, cib_diff);
1482         }
1483     }
1484 
1485     /* Always write to disk for successful ops with the flag set. This also
1486      * negates the need to detect ordering changes.
1487      */
1488     if ((rc == pcmk_ok)
1489         && pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1490 
1491         config_changed = true;
1492     }
1493 
1494     if ((rc == pcmk_ok)
1495         && !pcmk_any_flags_set(call_options, cib_dryrun|cib_transaction)) {
1496 
1497         if (result_cib != the_cib) {
1498             if (pcmk_is_set(operation->flags, cib__op_attr_writes_through)) {
1499                 config_changed = true;
1500             }
1501 
1502             crm_trace("Activating %s->%s%s",
1503                       crm_element_value(the_cib, PCMK_XA_NUM_UPDATES),
1504                       crm_element_value(result_cib, PCMK_XA_NUM_UPDATES),
1505                       (config_changed? " changed" : ""));
1506 
1507             rc = activateCibXml(result_cib, config_changed, op);
1508             if (rc != pcmk_ok) {
1509                 crm_err("Failed to activate new CIB: %s", pcmk_strerror(rc));
1510             }
1511         }
1512 
1513         if ((rc == pcmk_ok) && contains_config_change(*cib_diff)) {
1514             cib_read_config(config_hash, result_cib);
1515         }
1516 
1517         /* @COMPAT Nodes older than feature set 3.19.0 don't support
1518          * transactions. In a mixed-version cluster with nodes <3.19.0, we must
1519          * sync the updated CIB, so that the older nodes receive the changes.
1520          * Any node that has already applied the transaction will ignore the
1521          * synced CIB.
1522          *
1523          * To ensure the updated CIB is synced from only one node, we sync it
1524          * from the originator.
1525          */
1526         if ((operation->type == cib__op_commit_transact)
1527             && pcmk__str_eq(originator, OUR_NODENAME, pcmk__str_casei)
1528             && compare_version(crm_element_value(the_cib,
1529                                                  PCMK_XA_CRM_FEATURE_SET),
1530                                "3.19.0") < 0) {
1531 
1532             sync_our_cib(request, TRUE);
1533         }
1534 
1535         mainloop_timer_stop(digest_timer);
1536         mainloop_timer_start(digest_timer);
1537 
1538     } else if (rc == -pcmk_err_schema_validation) {
1539         pcmk__assert(result_cib != the_cib);
1540 
1541         if (output != NULL) {
1542             crm_log_xml_info(output, "cib:output");
1543             free_xml(output);
1544         }
1545 
1546         output = result_cib;
1547 
1548     } else {
1549         crm_trace("Not activating %d %d %s", rc,
1550                   pcmk_is_set(call_options, cib_dryrun),
1551                   crm_element_value(result_cib, PCMK_XA_NUM_UPDATES));
1552 
1553         if (result_cib != the_cib) {
1554             free_xml(result_cib);
1555         }
1556     }
1557 
1558     if (!pcmk_any_flags_set(call_options,
1559                             cib_dryrun|cib_inhibit_notify|cib_transaction)) {
1560         crm_trace("Sending notifications %d",
1561                   pcmk_is_set(call_options, cib_dryrun));
1562         cib_diff_notify(op, rc, call_id, client_id, client_name, originator,
1563                         input, *cib_diff);
1564     }
1565 
1566     pcmk__log_xml_patchset(LOG_TRACE, *cib_diff);
1567 
1568   done:
1569     if (!pcmk_is_set(call_options, cib_discard_reply) || cib_legacy_mode()) {
1570         *reply = create_cib_reply(op, call_id, client_id, call_options, rc,
1571                                   output);
1572     }
1573 
1574     if (output != the_cib) {
1575         free_xml(output);
1576     }
1577     crm_trace("done");
1578     return rc;
1579 }
1580 
1581 void
1582 cib_peer_callback(xmlNode * msg, void *private_data)
     /*  */
1583 {
1584     const char *reason = NULL;
1585     const char *originator = crm_element_value(msg, PCMK__XA_SRC);
1586 
1587     if (cib_legacy_mode()
1588         && pcmk__str_eq(originator, OUR_NODENAME,
1589                         pcmk__str_casei|pcmk__str_null_matches)) {
1590         /* message is from ourselves */
1591         int bcast_id = 0;
1592 
1593         if (crm_element_value_int(msg, PCMK__XA_CIB_LOCAL_NOTIFY_ID,
1594                                   &bcast_id) == 0) {
1595             check_local_notify(bcast_id);
1596         }
1597         return;
1598 
1599     } else if (crm_peer_cache == NULL) {
1600         reason = "membership not established";
1601         goto bail;
1602     }
1603 
1604     if (crm_element_value(msg, PCMK__XA_CIB_CLIENTNAME) == NULL) {
1605         crm_xml_add(msg, PCMK__XA_CIB_CLIENTNAME, originator);
1606     }
1607 
1608     /* crm_log_xml_trace(msg, "Peer[inbound]"); */
1609     cib_process_request(msg, TRUE, NULL);
1610     return;
1611 
1612   bail:
1613     if (reason) {
1614         const char *op = crm_element_value(msg, PCMK__XA_CIB_OP);
1615 
1616         crm_warn("Discarding %s message from %s: %s", op, originator, reason);
1617     }
1618 }
1619 
1620 static gboolean
1621 cib_force_exit(gpointer data)
     /*  */
1622 {
1623     crm_notice("Forcing exit!");
1624     terminate_cib(__func__, CRM_EX_ERROR);
1625     return FALSE;
1626 }
1627 
1628 static void
1629 disconnect_remote_client(gpointer key, gpointer value, gpointer user_data)
     /*  */
1630 {
1631     pcmk__client_t *a_client = value;
1632 
1633     crm_err("Can't disconnect client %s: Not implemented",
1634             pcmk__client_name(a_client));
1635 }
1636 
1637 static void
1638 initiate_exit(void)
     /*  */
1639 {
1640     int active = 0;
1641     xmlNode *leaving = NULL;
1642 
1643     active = pcmk__cluster_num_active_nodes();
1644     if (active < 2) { // This is the last active node
1645         terminate_cib(__func__, 0);
1646         return;
1647     }
1648 
1649     crm_info("Sending shutdown request to %d peers", active);
1650 
1651     leaving = pcmk__xe_create(NULL, PCMK__XE_EXIT_NOTIFICATION);
1652     crm_xml_add(leaving, PCMK__XA_T, PCMK__VALUE_CIB);
1653     crm_xml_add(leaving, PCMK__XA_CIB_OP, PCMK__CIB_REQUEST_SHUTDOWN);
1654 
1655     pcmk__cluster_send_message(NULL, crm_msg_cib, leaving);
1656     free_xml(leaving);
1657 
1658     g_timeout_add(EXIT_ESCALATION_MS, cib_force_exit, NULL);
1659 }
1660 
1661 void
1662 cib_shutdown(int nsig)
     /*  */
1663 {
1664     struct qb_ipcs_stats srv_stats;
1665 
1666     if (cib_shutdown_flag == FALSE) {
1667         int disconnects = 0;
1668         qb_ipcs_connection_t *c = NULL;
1669 
1670         cib_shutdown_flag = TRUE;
1671 
1672         c = qb_ipcs_connection_first_get(ipcs_rw);
1673         while (c != NULL) {
1674             qb_ipcs_connection_t *last = c;
1675 
1676             c = qb_ipcs_connection_next_get(ipcs_rw, last);
1677 
1678             crm_debug("Disconnecting r/w client %p...", last);
1679             qb_ipcs_disconnect(last);
1680             qb_ipcs_connection_unref(last);
1681             disconnects++;
1682         }
1683 
1684         c = qb_ipcs_connection_first_get(ipcs_ro);
1685         while (c != NULL) {
1686             qb_ipcs_connection_t *last = c;
1687 
1688             c = qb_ipcs_connection_next_get(ipcs_ro, last);
1689 
1690             crm_debug("Disconnecting r/o client %p...", last);
1691             qb_ipcs_disconnect(last);
1692             qb_ipcs_connection_unref(last);
1693             disconnects++;
1694         }
1695 
1696         c = qb_ipcs_connection_first_get(ipcs_shm);
1697         while (c != NULL) {
1698             qb_ipcs_connection_t *last = c;
1699 
1700             c = qb_ipcs_connection_next_get(ipcs_shm, last);
1701 
1702             crm_debug("Disconnecting non-blocking r/w client %p...", last);
1703             qb_ipcs_disconnect(last);
1704             qb_ipcs_connection_unref(last);
1705             disconnects++;
1706         }
1707 
1708         disconnects += pcmk__ipc_client_count();
1709 
1710         crm_debug("Disconnecting %d remote clients", pcmk__ipc_client_count());
1711         pcmk__foreach_ipc_client(disconnect_remote_client, NULL);
1712         crm_info("Disconnected %d clients", disconnects);
1713     }
1714 
1715     qb_ipcs_stats_get(ipcs_rw, &srv_stats, QB_FALSE);
1716 
1717     if (pcmk__ipc_client_count() == 0) {
1718         crm_info("All clients disconnected (%d)", srv_stats.active_connections);
1719         initiate_exit();
1720 
1721     } else {
1722         crm_info("Waiting on %d clients to disconnect (%d)",
1723                  pcmk__ipc_client_count(), srv_stats.active_connections);
1724     }
1725 }
1726 
1727 extern int remote_fd;
1728 extern int remote_tls_fd;
1729 
1730 /*!
1731  * \internal
1732  * \brief Close remote sockets, free the global CIB and quit
1733  *
1734  * \param[in] caller           Name of calling function (for log message)
1735  * \param[in] fast             If -1, skip disconnect; if positive, exit that
1736  */
1737 void
1738 terminate_cib(const char *caller, int fast)
     /*  */
1739 {
1740     crm_info("%s: Exiting%s...", caller,
1741              (fast > 0)? " fast" : mainloop ? " from mainloop" : "");
1742 
1743     if (remote_fd > 0) {
1744         close(remote_fd);
1745         remote_fd = 0;
1746     }
1747     if (remote_tls_fd > 0) {
1748         close(remote_tls_fd);
1749         remote_tls_fd = 0;
1750     }
1751 
1752     uninitializeCib();
1753 
1754     if (fast > 0) {
1755         /* Quit fast on error */
1756         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1757         crm_exit(fast);
1758 
1759     } else if ((mainloop != NULL) && g_main_loop_is_running(mainloop)) {
1760         /* Quit via returning from the main loop. If fast == -1, we skip the
1761          * disconnect here, and it will be done when the main loop returns
1762          * (this allows the peer status callback to avoid messing with the
1763          * peer caches).
1764          */
1765         if (fast == 0) {
1766             pcmk_cluster_disconnect(crm_cluster);
1767         }
1768         g_main_loop_quit(mainloop);
1769 
1770     } else {
1771         /* Quit via clean exit. Even the peer status callback can disconnect
1772          * here, because we're not returning control to the caller. */
1773         pcmk_cluster_disconnect(crm_cluster);
1774         pcmk__stop_based_ipc(ipcs_ro, ipcs_rw, ipcs_shm);
1775         crm_exit(CRM_EX_OK);
1776     }
1777 }
/* */
root/daemons/based/based_callbacks.c

DEFINITIONS
