root/crmd/join_dc.c

/* */

DEFINITIONS

1. crm_update_peer_join
2. initialize_join
3. create_dc_message
4. join_make_offer
5. do_dc_join_offer_all
6. do_dc_join_offer_one
7. compare_int_fields
8. do_dc_join_filter_offer
9. do_dc_join_finalize
10. finalize_sync_callback
11. join_update_complete_callback
12. do_dc_join_ack
13. finalize_join_for
14. check_join_state
15. do_dc_join_final
16. crmd_join_phase_count
17. crmd_join_phase_log

   1 /*
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include <crm_internal.h>
  19 
  20 #include <crm/crm.h>
  21 
  22 #include <crm/msg_xml.h>
  23 #include <crm/common/xml.h>
  24 #include <crm/cluster.h>
  25 
  26 #include <crmd_fsa.h>
  27 #include <crmd_messages.h>
  28 #include "tengine.h"
  29 
  30 char *max_epoch = NULL;
  31 char *max_generation_from = NULL;
  32 xmlNode *max_generation_xml = NULL;
  33 
  34 void initialize_join(gboolean before);
  35 void finalize_join_for(gpointer key, gpointer value, gpointer user_data);
  36 void finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data);
  37 gboolean check_join_state(enum crmd_fsa_state cur_state, const char *source);
  38 
  39 static int current_join_id = 0;
  40 unsigned long long saved_ccm_membership_id = 0;
  41 
  42 void
  43 crm_update_peer_join(const char *source, crm_node_t * node, enum crm_join_phase phase)
     /*  */
  44 {
  45     enum crm_join_phase last = 0;
  46 
  47     if(node == NULL) {
  48         crm_err("Could not update join because node not specified"
  49                 CRM_XS " join-%u source=%s phase=%s",
  50                 current_join_id, source, crm_join_phase_str(phase));
  51         return;
  52     }
  53 
  54     /* Remote nodes do not participate in joins */
  55     if (is_set(node->flags, crm_remote_node)) {
  56         return;
  57     }
  58 
  59     last = node->join;
  60 
  61     if(phase == last) {
  62         crm_trace("%s: Node %s[%u] - join-%u phase still %s",
  63                   source, node->uname, node->id, current_join_id,
  64                   crm_join_phase_str(last));
  65 
  66     } else if ((phase <= crm_join_none) || (phase == (last + 1))) {
  67         node->join = phase;
  68         crm_info("%s: Node %s[%u] - join-%u phase %s -> %s",
  69                  source, node->uname, node->id, current_join_id,
  70                  crm_join_phase_str(last), crm_join_phase_str(phase));
  71 
  72     } else {
  73         crm_err("Could not update join for node %s because phase transition invalid "
  74                 CRM_XS " join-%u source=%s node_id=%u last=%s new=%s",
  75                 node->uname, current_join_id, source, node->id,
  76                 crm_join_phase_str(last), crm_join_phase_str(phase));
  77     }
  78 }
  79 
  80 void
  81 initialize_join(gboolean before)
     /*  */
  82 {
  83     GHashTableIter iter;
  84     crm_node_t *peer = NULL;
  85 
  86     /* clear out/reset a bunch of stuff */
  87     crm_debug("join-%d: Initializing join data (flag=%s)",
  88               current_join_id, before ? "true" : "false");
  89 
  90     g_hash_table_iter_init(&iter, crm_peer_cache);
  91     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
  92         crm_update_peer_join(__FUNCTION__, peer, crm_join_none);
  93     }
  94 
  95     if (before) {
  96         if (max_generation_from != NULL) {
  97             free(max_generation_from);
  98             max_generation_from = NULL;
  99         }
 100         if (max_generation_xml != NULL) {
 101             free_xml(max_generation_xml);
 102             max_generation_xml = NULL;
 103         }
 104         clear_bit(fsa_input_register, R_HAVE_CIB);
 105         clear_bit(fsa_input_register, R_CIB_ASKED);
 106     }
 107 }
 108 
 109 /*!
 110  * \internal
 111  * \brief Create a join message from the DC
 112  *
 113  * \param[in] join_op  Join operation name
 114  * \param[in] host_to  Recipient of message
 115  */
 116 static xmlNode *
 117 create_dc_message(const char *join_op, const char *host_to)
     /*  */
 118 {
 119     xmlNode *msg = create_request(join_op, NULL, host_to, CRM_SYSTEM_CRMD,
 120                                   CRM_SYSTEM_DC, NULL);
 121 
 122     /* Identify which election this is a part of */
 123     crm_xml_add_int(msg, F_CRM_JOIN_ID, current_join_id);
 124 
 125     /* Add a field specifying whether the DC is shutting down. This keeps the
 126      * joining node from fencing the old DC if it becomes the new DC.
 127      */
 128     crm_xml_add_boolean(msg, F_CRM_DC_LEAVING,
 129                         is_set(fsa_input_register, R_SHUTDOWN));
 130     return msg;
 131 }
 132 
 133 static void
 134 join_make_offer(gpointer key, gpointer value, gpointer user_data)
     /*  */
 135 {
 136     xmlNode *offer = NULL;
 137     crm_node_t *member = (crm_node_t *)value;
 138 
 139     CRM_ASSERT(member != NULL);
 140     if (crm_is_peer_active(member) == FALSE) {
 141         crm_info("Not making an offer to %s: not active (%s)", member->uname, member->state);
 142         if(member->expected == NULL && safe_str_eq(member->state, CRM_NODE_LOST)) {
 143             /* You would think this unsafe, but in fact this plus an
 144              * active resource is what causes it to be fenced.
 145              *
 146              * Yes, this does mean that any node that dies at the same
 147              * time as the old DC and is not running resource (still)
 148              * won't be fenced.
 149              *
 150              * I'm not happy about this either.
 151              */
 152             crm_update_peer_expected(__FUNCTION__, member, CRMD_JOINSTATE_DOWN);
 153         }
 154         return;
 155     }
 156 
 157     if (member->uname == NULL) {
 158         crm_info("No recipient for welcome message.(Node uuid:%s)", member->uuid);
 159         return;
 160     }
 161 
 162     if (saved_ccm_membership_id != crm_peer_seq) {
 163         saved_ccm_membership_id = crm_peer_seq;
 164         crm_info("Making join offers based on membership %llu", crm_peer_seq);
 165     }
 166 
 167     if(user_data && member->join > crm_join_none) {
 168         crm_info("Skipping %s: already known %d", member->uname, member->join);
 169         return;
 170     }
 171 
 172     crm_update_peer_join(__FUNCTION__, (crm_node_t*)member, crm_join_none);
 173 
 174     offer = create_dc_message(CRM_OP_JOIN_OFFER, member->uname);
 175 
 176     /* send the welcome */
 177     crm_info("join-%d: Sending offer to %s", current_join_id, member->uname);
 178 
 179     send_cluster_message(member, crm_msg_crmd, offer, TRUE);
 180     free_xml(offer);
 181 
 182     crm_update_peer_join(__FUNCTION__, member, crm_join_welcomed);
 183     /* crm_update_peer_expected(__FUNCTION__, member, CRMD_JOINSTATE_PENDING); */
 184 }
 185 
 186 /*       A_DC_JOIN_OFFER_ALL    */
 187 void
 188 do_dc_join_offer_all(long long action,
     /*  */
 189                      enum crmd_fsa_cause cause,
 190                      enum crmd_fsa_state cur_state,
 191                      enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 192 {
 193     /* reset everyone's status back to down or in_ccm in the CIB
 194      *
 195      * any nodes that are active in the CIB but not in the CCM list
 196      *   will be seen as offline by the PE anyway
 197      */
 198     current_join_id++;
 199     initialize_join(TRUE);
 200 /*      do_update_cib_nodes(TRUE, __FUNCTION__); */
 201 
 202     update_dc(NULL);
 203     if (cause == C_HA_MESSAGE && current_input == I_NODE_JOIN) {
 204         crm_info("A new node joined the cluster");
 205     }
 206     g_hash_table_foreach(crm_peer_cache, join_make_offer, NULL);
 207 
 208     /* don't waste time by invoking the PE yet; */
 209     crm_info("join-%d: Waiting on %d outstanding join acks",
 210              current_join_id, crmd_join_phase_count(crm_join_welcomed));
 211 }
 212 
 213 /*       A_DC_JOIN_OFFER_ONE    */
 214 void
 215 do_dc_join_offer_one(long long action,
     /*  */
 216                      enum crmd_fsa_cause cause,
 217                      enum crmd_fsa_state cur_state,
 218                      enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 219 {
 220     crm_node_t *member;
 221     ha_msg_input_t *welcome = NULL;
 222 
 223     const char *op = NULL;
 224     const char *join_to = NULL;
 225 
 226     if (msg_data->data) {
 227         welcome = fsa_typed_data(fsa_dt_ha_msg);
 228 
 229     } else {
 230         crm_info("An unknown node joined - (re-)offer to any unconfirmed nodes");
 231         g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
 232         check_join_state(cur_state, __FUNCTION__);
 233         return;
 234     }
 235 
 236     if (welcome == NULL) {
 237         crm_err("Attempt to send welcome message without a message to reply to!");
 238         return;
 239     }
 240 
 241     join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
 242     if (join_to == NULL) {
 243         crm_err("Attempt to send welcome message without a host to reply to!");
 244         return;
 245     }
 246 
 247     member = crm_get_peer(0, join_to);
 248     op = crm_element_value(welcome->msg, F_CRM_TASK);
 249     if (join_to != NULL && (cur_state == S_INTEGRATION || cur_state == S_FINALIZE_JOIN)) {
 250         /* note: it _is_ possible that a node will have been
 251          *  sick or starting up when the original offer was made.
 252          *  however, it will either re-announce itself in due course
 253          *  _or_ we can re-store the original offer on the client.
 254          */
 255         crm_trace("(Re-)offering membership to %s...", join_to);
 256     }
 257 
 258     crm_info("join-%d: Processing %s request from %s in state %s",
 259              current_join_id, op, join_to, fsa_state2string(cur_state));
 260 
 261     crm_update_peer_join(__FUNCTION__, member, crm_join_none);
 262     join_make_offer(NULL, member, NULL);
 263 
 264     /* always offer to the DC (ourselves)
 265      * this ensures the correct value for max_generation_from
 266      */
 267     if (strcmp(join_to, fsa_our_uname) != 0) {
 268         member = crm_get_peer(0, fsa_our_uname);
 269         join_make_offer(NULL, member, NULL);
 270     }
 271 
 272     /* this was a genuine join request, cancel any existing
 273      * transition and invoke the PE
 274      */
 275     abort_transition(INFINITY, tg_restart, "Node join", NULL);
 276 
 277     /* don't waste time by invoking the PE yet; */
 278     crm_debug("Waiting on %d outstanding join acks for join-%d",
 279               crmd_join_phase_count(crm_join_welcomed), current_join_id);
 280 }
 281 
 282 static int
 283 compare_int_fields(xmlNode * left, xmlNode * right, const char *field)
     /*  */
 284 {
 285     const char *elem_l = crm_element_value(left, field);
 286     const char *elem_r = crm_element_value(right, field);
 287 
 288     int int_elem_l = crm_int_helper(elem_l, NULL);
 289     int int_elem_r = crm_int_helper(elem_r, NULL);
 290 
 291     if (int_elem_l < int_elem_r) {
 292         return -1;
 293 
 294     } else if (int_elem_l > int_elem_r) {
 295         return 1;
 296     }
 297 
 298     return 0;
 299 }
 300 
 301 /*       A_DC_JOIN_PROCESS_REQ  */
 302 void
 303 do_dc_join_filter_offer(long long action,
     /*  */
 304                         enum crmd_fsa_cause cause,
 305                         enum crmd_fsa_state cur_state,
 306                         enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 307 {
 308     xmlNode *generation = NULL;
 309 
 310     int cmp = 0;
 311     int join_id = -1;
 312     gboolean ack_nack_bool = TRUE;
 313     const char *ack_nack = CRMD_JOINSTATE_MEMBER;
 314     ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
 315 
 316     const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
 317     const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);
 318 
 319     crm_node_t *join_node = crm_get_peer(0, join_from);
 320 
 321     crm_debug("Processing req from %s", join_from);
 322 
 323     generation = join_ack->xml;
 324     crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
 325 
 326     if (max_generation_xml != NULL && generation != NULL) {
 327         int lpc = 0;
 328 
 329         const char *attributes[] = {
 330             XML_ATTR_GENERATION_ADMIN,
 331             XML_ATTR_GENERATION,
 332             XML_ATTR_NUMUPDATES,
 333         };
 334 
 335         for (lpc = 0; cmp == 0 && lpc < DIMOF(attributes); lpc++) {
 336             cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
 337         }
 338     }
 339 
 340     if (join_id != current_join_id) {
 341         crm_debug("Invalid response from %s: join-%d vs. join-%d",
 342                   join_from, join_id, current_join_id);
 343         check_join_state(cur_state, __FUNCTION__);
 344         return;
 345 
 346     } else if (join_node == NULL || crm_is_peer_active(join_node) == FALSE) {
 347         crm_err("Node %s is not a member", join_from);
 348         ack_nack_bool = FALSE;
 349 
 350     } else if (generation == NULL) {
 351         crm_err("Generation was NULL");
 352         ack_nack_bool = FALSE;
 353 
 354     } else if (max_generation_xml == NULL) {
 355         max_generation_xml = copy_xml(generation);
 356         max_generation_from = strdup(join_from);
 357 
 358     } else if (cmp < 0 || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) {
 359         crm_debug("%s has a better generation number than"
 360                   " the current max %s", join_from, max_generation_from);
 361         if (max_generation_xml) {
 362             crm_log_xml_debug(max_generation_xml, "Max generation");
 363         }
 364         crm_log_xml_debug(generation, "Their generation");
 365 
 366         free(max_generation_from);
 367         free_xml(max_generation_xml);
 368 
 369         max_generation_from = strdup(join_from);
 370         max_generation_xml = copy_xml(join_ack->xml);
 371     }
 372 
 373     if (ack_nack_bool == FALSE) {
 374         /* NACK this client */
 375         ack_nack = CRMD_JOINSTATE_NACK;
 376         crm_update_peer_join(__FUNCTION__, join_node, crm_join_nack);
 377         crm_err("Rejecting cluster join request from %s " CRM_XS
 378                 " NACK join-%d ref=%s", join_from, join_id, ref);
 379 
 380     } else {
 381         crm_debug("join-%d: Welcoming node %s (ref %s)", join_id, join_from, ref);
 382         crm_update_peer_join(__FUNCTION__, join_node, crm_join_integrated);
 383     }
 384 
 385     crm_update_peer_expected(__FUNCTION__, join_node, ack_nack);
 386 
 387     crm_debug("%u nodes have been integrated into join-%d",
 388               crmd_join_phase_count(crm_join_integrated), join_id);
 389 
 390 
 391     if (check_join_state(cur_state, __FUNCTION__) == FALSE) {
 392         /* don't waste time by invoking the PE yet; */
 393         crm_debug("join-%d: Still waiting on %d outstanding offers",
 394                   join_id, crmd_join_phase_count(crm_join_welcomed));
 395     }
 396 }
 397 
 398 /*      A_DC_JOIN_FINALIZE      */
 399 void
 400 do_dc_join_finalize(long long action,
     /*  */
 401                     enum crmd_fsa_cause cause,
 402                     enum crmd_fsa_state cur_state,
 403                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 404 {
 405     char *sync_from = NULL;
 406     int rc = pcmk_ok;
 407 
 408     /* This we can do straight away and avoid clients timing us out
 409      *  while we compute the latest CIB
 410      */
 411     crm_debug("Finalizing join-%d for %d clients",
 412               current_join_id, crmd_join_phase_count(crm_join_integrated));
 413 
 414     crmd_join_phase_log(LOG_INFO);
 415     if (crmd_join_phase_count(crm_join_welcomed) != 0) {
 416         crm_info("Waiting for %d more nodes", crmd_join_phase_count(crm_join_welcomed));
 417         /* crmd_fsa_stall(FALSE); Needed? */
 418         return;
 419 
 420     } else if (crmd_join_phase_count(crm_join_integrated) == 0) {
 421         /* Nothing to do */
 422         check_join_state(fsa_state, __FUNCTION__);
 423         return;
 424     }
 425 
 426     clear_bit(fsa_input_register, R_HAVE_CIB);
 427     if (max_generation_from == NULL || safe_str_eq(max_generation_from, fsa_our_uname)) {
 428         set_bit(fsa_input_register, R_HAVE_CIB);
 429     }
 430 
 431     if (is_set(fsa_input_register, R_IN_TRANSITION)) {
 432         crm_warn("Delaying response to cluster join offer while transition in progress "
 433                  CRM_XS " join-%d", current_join_id);
 434         crmd_fsa_stall(FALSE);
 435         return;
 436     }
 437 
 438     if (max_generation_from && is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
 439         /* ask for the agreed best CIB */
 440         sync_from = strdup(max_generation_from);
 441         set_bit(fsa_input_register, R_CIB_ASKED);
 442         crm_notice("Syncing the Cluster Information Base from %s to rest of cluster "
 443                    CRM_XS " join-%d", sync_from, current_join_id);
 444         crm_log_xml_notice(max_generation_xml, "Requested version");
 445 
 446     } else {
 447         /* Send _our_ CIB out to everyone */
 448         sync_from = strdup(fsa_our_uname);
 449         crm_info("join-%d: Syncing our CIB to the rest of the cluster",
 450                  current_join_id);
 451         crm_log_xml_debug(max_generation_xml, "Requested version");
 452     }
 453 
 454 
 455     rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override);
 456     fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback);
 457 }
 458 
 459 void
 460 finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 461 {
 462     CRM_LOG_ASSERT(-EPERM != rc);
 463     clear_bit(fsa_input_register, R_CIB_ASKED);
 464     if (rc != pcmk_ok) {
 465         do_crm_log((rc == -pcmk_err_old_data ? LOG_WARNING : LOG_ERR),
 466                    "Sync from %s failed: %s", (char *)user_data, pcmk_strerror(rc));
 467 
 468         /* restart the whole join process */
 469         register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL, __FUNCTION__);
 470 
 471     } else if (AM_I_DC && fsa_state == S_FINALIZE_JOIN) {
 472         set_bit(fsa_input_register, R_HAVE_CIB);
 473         clear_bit(fsa_input_register, R_CIB_ASKED);
 474 
 475         /* make sure dc_uuid is re-set to us */
 476         if (check_join_state(fsa_state, __FUNCTION__) == FALSE) {
 477             crm_debug("Notifying %d clients of join-%d results",
 478                       crmd_join_phase_count(crm_join_integrated), current_join_id);
 479             g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
 480         }
 481 
 482     } else {
 483         crm_debug("No longer the DC in S_FINALIZE_JOIN: %s/%s",
 484                   AM_I_DC ? "DC" : "CRMd", fsa_state2string(fsa_state));
 485     }
 486 }
 487 
 488 static void
 489 join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 490 {
 491     fsa_data_t *msg_data = NULL;
 492 
 493     if (rc == pcmk_ok) {
 494         crm_debug("Join update %d complete", call_id);
 495         check_join_state(fsa_state, __FUNCTION__);
 496 
 497     } else {
 498         crm_err("Join update %d failed", call_id);
 499         crm_log_xml_debug(msg, "failed");
 500         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 501     }
 502 }
 503 
 504 /*      A_DC_JOIN_PROCESS_ACK   */
 505 void
 506 do_dc_join_ack(long long action,
     /*  */
 507                enum crmd_fsa_cause cause,
 508                enum crmd_fsa_state cur_state,
 509                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 510 {
 511     int join_id = -1;
 512     int call_id = 0;
 513     ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);
 514 
 515     const char *op = crm_element_value(join_ack->msg, F_CRM_TASK);
 516     const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
 517     crm_node_t *peer = crm_get_peer(0, join_from);
 518 
 519     if (safe_str_neq(op, CRM_OP_JOIN_CONFIRM) || peer == NULL) {
 520         crm_debug("Ignoring op=%s message from %s", op, join_from);
 521         return;
 522     }
 523 
 524     crm_trace("Processing ack from %s", join_from);
 525     crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);
 526 
 527     if (peer->join != crm_join_finalized) {
 528         crm_info("Join not in progress: ignoring join-%d from %s (phase = %d)",
 529                  join_id, join_from, peer->join);
 530         return;
 531 
 532     } else if (join_id != current_join_id) {
 533         crm_err("Invalid response from %s: join-%d vs. join-%d",
 534                 join_from, join_id, current_join_id);
 535         crm_update_peer_join(__FUNCTION__, peer, crm_join_nack);
 536         return;
 537     }
 538 
 539     crm_update_peer_join(__FUNCTION__, peer, crm_join_confirmed);
 540 
 541     crm_info("join-%d: Updating node state to %s for %s",
 542              join_id, CRMD_JOINSTATE_MEMBER, join_from);
 543 
 544     /* update CIB with the current LRM status from the node
 545      * We don't need to notify the TE of these updates, a transition will
 546      *   be started in due time
 547      */
 548     erase_status_tag(join_from, XML_CIB_TAG_LRM, cib_scope_local);
 549 
 550     if (safe_str_eq(join_from, fsa_our_uname)) {
 551         xmlNode *now_dc_lrmd_state = do_lrm_query(TRUE, fsa_our_uname);
 552 
 553         if (now_dc_lrmd_state != NULL) {
 554             crm_debug("LRM state is updated from do_lrm_query.(%s)", join_from);
 555             fsa_cib_update(XML_CIB_TAG_STATUS, now_dc_lrmd_state,
 556                 cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
 557             free_xml(now_dc_lrmd_state);
 558         } else {
 559             crm_warn("Could not get our LRM state. LRM state is updated from join_ack->xml.(%s)", join_from);
 560             fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
 561                 cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
 562         }
 563     } else {
 564         crm_debug("LRM state is updated from join_ack->xml.(%s)", join_from);
 565         fsa_cib_update(XML_CIB_TAG_STATUS, join_ack->xml,
 566            cib_scope_local | cib_quorum_override | cib_can_create, call_id, NULL);
 567     }
 568 
 569     fsa_register_cib_callback(call_id, FALSE, NULL, join_update_complete_callback);
 570     crm_debug("join-%d: Registered callback for LRM update %d", join_id, call_id);
 571 }
 572 
 573 void
 574 finalize_join_for(gpointer key, gpointer value, gpointer user_data)
     /*  */
 575 {
 576     xmlNode *acknak = NULL;
 577     xmlNode *tmp1 = NULL;
 578     crm_node_t *join_node = value;
 579     const char *join_to = join_node->uname;
 580 
 581     if(join_node->join != crm_join_integrated) {
 582         crm_trace("Skipping %s in state %d", join_to, join_node->join);
 583         return;
 584     }
 585 
 586     /* make sure a node entry exists for the new node */
 587     crm_trace("Creating node entry for %s", join_to);
 588 
 589     tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
 590     set_uuid(tmp1, XML_ATTR_UUID, join_node);
 591     crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
 592 
 593     fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1,
 594                         cib_scope_local | cib_quorum_override | cib_can_create);
 595     free_xml(tmp1);
 596 
 597     join_node = crm_get_peer(0, join_to);
 598     if (crm_is_peer_active(join_node) == FALSE) {
 599         /*
 600          * NACK'ing nodes that the membership layer doesn't know about yet
 601          * simply creates more churn
 602          *
 603          * Better to leave them waiting and let the join restart when
 604          * the new membership event comes in
 605          *
 606          * All other NACKs (due to versions etc) should still be processed
 607          */
 608         crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_PENDING);
 609         return;
 610     }
 611 
 612     /* send the ack/nack to the node */
 613     acknak = create_dc_message(CRM_OP_JOIN_ACKNAK, join_to);
 614 
 615     crm_debug("join-%d: ACK'ing join request from %s",
 616               current_join_id, join_to);
 617     crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE);
 618     crm_update_peer_join(__FUNCTION__, join_node, crm_join_finalized);
 619     crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER);
 620 
 621     send_cluster_message(crm_get_peer(0, join_to), crm_msg_crmd, acknak, TRUE);
 622     free_xml(acknak);
 623     return;
 624 }
 625 
 626 void ghash_print_node(gpointer key, gpointer value, gpointer user_data);
 627 
 628 gboolean
 629 check_join_state(enum crmd_fsa_state cur_state, const char *source)
     /*  */
 630 {
 631     static unsigned long long highest_seq = 0;
 632 
 633     crm_debug("Invoked by %s in state: %s", source, fsa_state2string(cur_state));
 634 
 635     if (saved_ccm_membership_id != crm_peer_seq) {
 636         crm_debug("%s: Membership changed since join started: %llu -> %llu (%llu)",
 637                   source, saved_ccm_membership_id, crm_peer_seq, highest_seq);
 638         if(highest_seq < crm_peer_seq) {
 639             /* Don't spam the FSA with duplicates */
 640             highest_seq = crm_peer_seq;
 641             register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
 642         }
 643 
 644     } else if (cur_state == S_INTEGRATION) {
 645         if (crmd_join_phase_count(crm_join_welcomed) == 0) {
 646             crm_debug("join-%d: Integration of %d peers complete: %s",
 647                       current_join_id, crmd_join_phase_count(crm_join_integrated), source);
 648             register_fsa_input_before(C_FSA_INTERNAL, I_INTEGRATED, NULL);
 649             return TRUE;
 650         }
 651 
 652     } else if (cur_state == S_FINALIZE_JOIN) {
 653         if (is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
 654             crm_debug("join-%d: Delaying I_FINALIZED until we have the CIB", current_join_id);
 655             return TRUE;
 656 
 657         } else if (crmd_join_phase_count(crm_join_welcomed) != 0) {
 658             crm_debug("join-%d: Still waiting on %d welcomed nodes",
 659                       current_join_id, crmd_join_phase_count(crm_join_welcomed));
 660             crmd_join_phase_log(LOG_DEBUG);
 661 
 662         } else if (crmd_join_phase_count(crm_join_integrated) != 0) {
 663             crm_debug("join-%d: Still waiting on %d integrated nodes",
 664                       current_join_id, crmd_join_phase_count(crm_join_integrated));
 665             crmd_join_phase_log(LOG_DEBUG);
 666 
 667         } else if (crmd_join_phase_count(crm_join_finalized) != 0) {
 668             crm_debug("join-%d: Still waiting on %d finalized nodes",
 669                       current_join_id, crmd_join_phase_count(crm_join_finalized));
 670             crmd_join_phase_log(LOG_DEBUG);
 671 
 672         } else {
 673             crm_debug("join-%d complete: %s", current_join_id, source);
 674             register_fsa_input_later(C_FSA_INTERNAL, I_FINALIZED, NULL);
 675             return TRUE;
 676         }
 677     }
 678 
 679     return FALSE;
 680 }
 681 
 682 void
 683 do_dc_join_final(long long action,
     /*  */
 684                  enum crmd_fsa_cause cause,
 685                  enum crmd_fsa_state cur_state,
 686                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 687 {
 688     crm_debug("Ensuring DC, quorum and node attributes are up-to-date");
 689 #if !HAVE_ATOMIC_ATTRD
 690     /* Ask attrd to write all attributes to disk. This is not needed for
 691      * atomic attrd because atomic attrd does a peer sync and write-out
 692      * when winning an election.
 693      */
 694     update_attrd(NULL, NULL, NULL, NULL, FALSE);
 695 #endif
 696     crm_update_quorum(crm_have_quorum, TRUE);
 697 }
 698 
 699 int crmd_join_phase_count(enum crm_join_phase phase)
     /*  */
 700 {
 701     int count = 0;
 702     crm_node_t *peer;
 703     GHashTableIter iter;
 704 
 705     g_hash_table_iter_init(&iter, crm_peer_cache);
 706     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
 707         if(peer->join == phase) {
 708             count++;
 709         }
 710     }
 711     return count;
 712 }
 713 
 714 void crmd_join_phase_log(int level)
     /*  */
 715 {
 716     crm_node_t *peer;
 717     GHashTableIter iter;
 718 
 719     g_hash_table_iter_init(&iter, crm_peer_cache);
 720     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &peer)) {
 721         do_crm_log(level, "join-%d: %s=%s", current_join_id, peer->uname,
 722                    crm_join_phase_str(peer->join));
 723     }
 724 }

/* */