daemons/controld/controld

/* */
This source file includes following definitions.
init_dotfile
do_fsa_action
do_log
s_crmd_fsa
s_crmd_fsa_actions
log_fsa_input
check_join_counts
do_state_transition
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <stdint.h>                 // uint64_t
  15 #include <string.h>
  16 #include <time.h>
  17 
  18 #include <crm/crm.h>
  19 #include <crm/lrmd.h>
  20 #include <crm/cib.h>
  21 #include <crm/msg_xml.h>
  22 #include <crm/common/xml.h>
  23 #include <crm/cluster/election_internal.h>
  24 #include <crm/cluster.h>
  25 
  26 #include <pacemaker-controld.h>
  27 #include <controld_matrix.h>
  28 
  29 char *fsa_our_dc = NULL;
  30 cib_t *fsa_cib_conn = NULL;
  31 char *fsa_our_dc_version = NULL;
  32 
  33 char *fsa_our_uuid = NULL;
  34 char *fsa_our_uname = NULL;
  35 
  36 char *fsa_cluster_name = NULL;
  37 
  38 gboolean do_fsa_stall = FALSE;
  39 uint64_t fsa_input_register = 0;
  40 uint64_t fsa_actions = A_NOTHING;
  41 enum crmd_fsa_state fsa_state = S_STARTING;
  42 
  43 extern uint highest_born_on;
  44 extern uint num_join_invites;
  45 
  46 #define DOT_PREFIX "actions:trace: "
  47 #define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
  48 
  49 static void do_state_transition(enum crmd_fsa_state cur_state,
  50                                 enum crmd_fsa_state next_state,
  51                                 fsa_data_t *msg_data);
  52 
  53 void s_crmd_fsa_actions(fsa_data_t * fsa_data);
  54 void log_fsa_input(fsa_data_t * stored_msg);
  55 void init_dotfile(void);
  56 
  57 void
  58 init_dotfile(void)
     /*  */
  59 {
  60     do_dot_log(DOT_PREFIX "digraph \"g\" {");
  61     do_dot_log(DOT_PREFIX "     size = \"30,30\"");
  62     do_dot_log(DOT_PREFIX "     graph [");
  63     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  64     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  65     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  66     do_dot_log(DOT_PREFIX "             bb = \"0,0,398.922306,478.927856\"");
  67     do_dot_log(DOT_PREFIX "             color = \"black\"");
  68     do_dot_log(DOT_PREFIX "     ]");
  69     do_dot_log(DOT_PREFIX "     node [");
  70     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  71     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  72     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  73     do_dot_log(DOT_PREFIX "             shape = \"ellipse\"");
  74     do_dot_log(DOT_PREFIX "             color = \"black\"");
  75     do_dot_log(DOT_PREFIX "     ]");
  76     do_dot_log(DOT_PREFIX "     edge [");
  77     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  78     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  79     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  80     do_dot_log(DOT_PREFIX "             color = \"black\"");
  81     do_dot_log(DOT_PREFIX "     ]");
  82     do_dot_log(DOT_PREFIX "// special nodes");
  83     do_dot_log(DOT_PREFIX "     \"S_PENDING\" ");
  84     do_dot_log(DOT_PREFIX "     [");
  85     do_dot_log(DOT_PREFIX "      color = \"blue\"");
  86     do_dot_log(DOT_PREFIX "      fontcolor = \"blue\"");
  87     do_dot_log(DOT_PREFIX "      ]");
  88     do_dot_log(DOT_PREFIX "     \"S_TERMINATE\" ");
  89     do_dot_log(DOT_PREFIX "     [");
  90     do_dot_log(DOT_PREFIX "      color = \"red\"");
  91     do_dot_log(DOT_PREFIX "      fontcolor = \"red\"");
  92     do_dot_log(DOT_PREFIX "      ]");
  93     do_dot_log(DOT_PREFIX "// DC only nodes");
  94     do_dot_log(DOT_PREFIX "     \"S_INTEGRATION\" [ fontcolor = \"green\" ]");
  95     do_dot_log(DOT_PREFIX "     \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
  96     do_dot_log(DOT_PREFIX "     \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
  97     do_dot_log(DOT_PREFIX "     \"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
  98     do_dot_log(DOT_PREFIX "     \"S_IDLE\" [ fontcolor = \"green\" ]");
  99 }
 100 
 101 static void
 102 do_fsa_action(fsa_data_t * fsa_data, long long an_action,
     /*  */
 103               void (*function) (long long action,
 104                                 enum crmd_fsa_cause cause,
 105                                 enum crmd_fsa_state cur_state,
 106                                 enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
 107 {
 108     controld_clear_fsa_action_flags(an_action);
 109     crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
 110     function(an_action, fsa_data->fsa_cause, fsa_state, fsa_data->fsa_input, fsa_data);
 111 }
 112 
 113 static const uint64_t startup_actions =
 114     A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
 115     A_STARTED | A_CL_JOIN_QUERY;
 116 
 117 // A_LOG, A_WARN, A_ERROR
 118 void
 119 do_log(long long action, enum crmd_fsa_cause cause,
     /*  */
 120        enum crmd_fsa_state cur_state,
 121        enum crmd_fsa_input current_input, fsa_data_t *msg_data)
 122 {
 123     unsigned log_type = LOG_TRACE;
 124 
 125     if (action & A_LOG) {
 126         log_type = LOG_INFO;
 127     } else if (action & A_WARN) {
 128         log_type = LOG_WARNING;
 129     } else if (action & A_ERROR) {
 130         log_type = LOG_ERR;
 131     }
 132 
 133     do_crm_log(log_type, "Input %s received in state %s from %s",
 134                fsa_input2string(msg_data->fsa_input),
 135                fsa_state2string(cur_state), msg_data->origin);
 136 
 137     if (msg_data->data_type == fsa_dt_ha_msg) {
 138         ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
 139 
 140         crm_log_xml_debug(input->msg, __func__);
 141 
 142     } else if (msg_data->data_type == fsa_dt_xml) {
 143         xmlNode *input = fsa_typed_data(msg_data->data_type);
 144 
 145         crm_log_xml_debug(input, __func__);
 146 
 147     } else if (msg_data->data_type == fsa_dt_lrm) {
 148         lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
 149 
 150         do_crm_log(log_type,
 151                    "Resource %s: Call ID %d returned %d (%d)."
 152                    "  New status if rc=0: %s",
 153                    input->rsc_id, input->call_id, input->rc,
 154                    input->op_status, (char *)input->user_data);
 155     }
 156 }
 157 
 158 enum crmd_fsa_state
 159 s_crmd_fsa(enum crmd_fsa_cause cause)
     /*  */
 160 {
 161     fsa_data_t *fsa_data = NULL;
 162     uint64_t register_copy = fsa_input_register;
 163     uint64_t new_actions = A_NOTHING;
 164     enum crmd_fsa_state last_state;
 165 
 166     crm_trace("FSA invoked with Cause: %s\tState: %s",
 167               fsa_cause2string(cause), fsa_state2string(fsa_state));
 168 
 169     fsa_dump_actions(fsa_actions, "Initial");
 170 
 171     do_fsa_stall = FALSE;
 172     if ((fsa_message_queue == NULL) && (fsa_actions != A_NOTHING)) {
 173         /* fake the first message so we can get into the loop */
 174         fsa_data = calloc(1, sizeof(fsa_data_t));
 175         fsa_data->fsa_input = I_NULL;
 176         fsa_data->fsa_cause = C_FSA_INTERNAL;
 177         fsa_data->origin = __func__;
 178         fsa_data->data_type = fsa_dt_none;
 179         fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
 180         fsa_data = NULL;
 181     }
 182     while ((fsa_message_queue != NULL) && !do_fsa_stall) {
 183         crm_trace("Checking messages (%d remaining)", g_list_length(fsa_message_queue));
 184 
 185         fsa_data = get_message();
 186         if(fsa_data == NULL) {
 187             continue;
 188         }
 189 
 190         log_fsa_input(fsa_data);
 191 
 192         /* add any actions back to the queue */
 193         controld_set_fsa_action_flags(fsa_data->actions);
 194         fsa_dump_actions(fsa_data->actions, "Restored actions");
 195 
 196         /* get the next batch of actions */
 197         new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state];
 198         controld_set_fsa_action_flags(new_actions);
 199         fsa_dump_actions(new_actions, "New actions");
 200 
 201         if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
 202             crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
 203                       fsa_input2string(fsa_data->fsa_input),
 204                       fsa_state2string(fsa_state),
 205                       fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
 206         }
 207 
 208         /* logging : *before* the state is changed */
 209         if (pcmk_is_set(fsa_actions, A_ERROR)) {
 210             do_fsa_action(fsa_data, A_ERROR, do_log);
 211         }
 212         if (pcmk_is_set(fsa_actions, A_WARN)) {
 213             do_fsa_action(fsa_data, A_WARN, do_log);
 214         }
 215         if (pcmk_is_set(fsa_actions, A_LOG)) {
 216             do_fsa_action(fsa_data, A_LOG, do_log);
 217         }
 218 
 219         /* update state variables */
 220         last_state = fsa_state;
 221         fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state];
 222 
 223         /*
 224          * Remove certain actions during shutdown
 225          */
 226         if (fsa_state == S_STOPPING || ((fsa_input_register & R_SHUTDOWN) == R_SHUTDOWN)) {
 227             controld_clear_fsa_action_flags(startup_actions);
 228         }
 229 
 230         /*
 231          * Hook for change of state.
 232          * Allows actions to be added or removed when entering a state
 233          */
 234         if (last_state != fsa_state) {
 235             do_state_transition(last_state, fsa_state, fsa_data);
 236         } else {
 237             do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
 238                        " \tInput=%s \tOrigin=%s() \tid=%d",
 239                        fsa_state2string(fsa_state),
 240                        fsa_cause2string(fsa_data->fsa_cause),
 241                        fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
 242         }
 243 
 244         /* start doing things... */
 245         s_crmd_fsa_actions(fsa_data);
 246         delete_fsa_input(fsa_data);
 247         fsa_data = NULL;
 248     }
 249 
 250     if ((fsa_message_queue != NULL) || (fsa_actions != A_NOTHING)
 251         || do_fsa_stall) {
 252         crm_debug("Exiting the FSA: queue=%d, fsa_actions=0x%llx, stalled=%s",
 253                   g_list_length(fsa_message_queue),
 254                   (unsigned long long) fsa_actions, pcmk__btoa(do_fsa_stall));
 255     } else {
 256         crm_trace("Exiting the FSA");
 257     }
 258 
 259     /* cleanup inputs? */
 260     if (register_copy != fsa_input_register) {
 261         uint64_t same = register_copy & fsa_input_register;
 262 
 263         fsa_dump_inputs(LOG_DEBUG, "Added", fsa_input_register ^ same);
 264         fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
 265     }
 266 
 267     fsa_dump_actions(fsa_actions, "Remaining");
 268     fsa_dump_queue(LOG_DEBUG);
 269 
 270     return fsa_state;
 271 }
 272 
 273 void
 274 s_crmd_fsa_actions(fsa_data_t * fsa_data)
     /*  */
 275 {
 276     /*
 277      * Process actions in order of priority but do only one
 278      * action at a time to avoid complicating the ordering.
 279      */
 280     CRM_CHECK(fsa_data != NULL, return);
 281     while (fsa_actions != A_NOTHING && do_fsa_stall == FALSE) {
 282 
 283         /* regular action processing in order of action priority
 284          *
 285          * Make sure all actions that connect to required systems
 286          * are performed first
 287          */
 288         if (fsa_actions & A_ERROR) {
 289             do_fsa_action(fsa_data, A_ERROR, do_log);
 290         } else if (fsa_actions & A_WARN) {
 291             do_fsa_action(fsa_data, A_WARN, do_log);
 292         } else if (fsa_actions & A_LOG) {
 293             do_fsa_action(fsa_data, A_LOG, do_log);
 294 
 295             /* get out of here NOW! before anything worse happens */
 296         } else if (fsa_actions & A_EXIT_1) {
 297             do_fsa_action(fsa_data, A_EXIT_1, do_exit);
 298 
 299             /* sub-system restart */
 300         } else if ((fsa_actions & O_LRM_RECONNECT) == O_LRM_RECONNECT) {
 301             do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
 302         } else if ((fsa_actions & O_CIB_RESTART) == O_CIB_RESTART) {
 303             do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
 304         } else if ((fsa_actions & O_PE_RESTART) == O_PE_RESTART) {
 305             do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
 306         } else if ((fsa_actions & O_TE_RESTART) == O_TE_RESTART) {
 307             do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
 308 
 309             /* essential start tasks */
 310         } else if (fsa_actions & A_STARTUP) {
 311             do_fsa_action(fsa_data, A_STARTUP, do_startup);
 312         } else if (fsa_actions & A_CIB_START) {
 313             do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
 314         } else if (fsa_actions & A_HA_CONNECT) {
 315             do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
 316         } else if (fsa_actions & A_READCONFIG) {
 317             do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
 318 
 319             /* sub-system start/connect */
 320         } else if (fsa_actions & A_LRM_CONNECT) {
 321             do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
 322 
 323         } else if (fsa_actions & A_TE_START) {
 324             do_fsa_action(fsa_data, A_TE_START, do_te_control);
 325         } else if (fsa_actions & A_PE_START) {
 326             do_fsa_action(fsa_data, A_PE_START, do_pe_control);
 327 
 328             /* Timers */
 329 /*              else if(fsa_actions & O_DC_TIMER_RESTART) {
 330                 do_fsa_action(fsa_data, O_DC_TIMER_RESTART,          do_timer_control) */ ;
 331         } else if (fsa_actions & A_DC_TIMER_STOP) {
 332             do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
 333         } else if (fsa_actions & A_INTEGRATE_TIMER_STOP) {
 334             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
 335         } else if (fsa_actions & A_INTEGRATE_TIMER_START) {
 336             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
 337         } else if (fsa_actions & A_FINALIZE_TIMER_STOP) {
 338             do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
 339         } else if (fsa_actions & A_FINALIZE_TIMER_START) {
 340             do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
 341 
 342             /*
 343              * Highest priority actions
 344              */
 345         } else if (fsa_actions & A_MSG_ROUTE) {
 346             do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
 347         } else if (fsa_actions & A_RECOVER) {
 348             do_fsa_action(fsa_data, A_RECOVER, do_recover);
 349         } else if (fsa_actions & A_CL_JOIN_RESULT) {
 350             do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond);
 351         } else if (fsa_actions & A_CL_JOIN_REQUEST) {
 352             do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond);
 353         } else if (fsa_actions & A_SHUTDOWN_REQ) {
 354             do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
 355         } else if (fsa_actions & A_ELECTION_VOTE) {
 356             do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
 357         } else if (fsa_actions & A_ELECTION_COUNT) {
 358             do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
 359         } else if (fsa_actions & A_LRM_EVENT) {
 360             do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
 361 
 362             /*
 363              * High priority actions
 364              */
 365         } else if (fsa_actions & A_STARTED) {
 366             do_fsa_action(fsa_data, A_STARTED, do_started);
 367         } else if (fsa_actions & A_CL_JOIN_QUERY) {
 368             do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
 369         } else if (fsa_actions & A_DC_TIMER_START) {
 370             do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
 371 
 372             /*
 373              * Medium priority actions
 374              * - Membership
 375              */
 376         } else if (fsa_actions & A_DC_TAKEOVER) {
 377             do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
 378         } else if (fsa_actions & A_DC_RELEASE) {
 379             do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
 380         } else if (fsa_actions & A_DC_JOIN_FINAL) {
 381             do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
 382         } else if (fsa_actions & A_ELECTION_CHECK) {
 383             do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
 384         } else if (fsa_actions & A_ELECTION_START) {
 385             do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
 386         } else if (fsa_actions & A_DC_JOIN_OFFER_ALL) {
 387             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
 388         } else if (fsa_actions & A_DC_JOIN_OFFER_ONE) {
 389             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
 390         } else if (fsa_actions & A_DC_JOIN_PROCESS_REQ) {
 391             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer);
 392         } else if (fsa_actions & A_DC_JOIN_PROCESS_ACK) {
 393             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
 394         } else if (fsa_actions & A_DC_JOIN_FINALIZE) {
 395             do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
 396         } else if (fsa_actions & A_CL_JOIN_ANNOUNCE) {
 397             do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
 398 
 399             /*
 400              * Low(er) priority actions
 401              * Make sure the CIB is always updated before invoking the
 402              * scheduler, and the scheduler before the transition engine.
 403              */
 404         } else if (fsa_actions & A_TE_HALT) {
 405             do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
 406         } else if (fsa_actions & A_TE_CANCEL) {
 407             do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
 408         } else if (fsa_actions & A_LRM_INVOKE) {
 409             do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
 410         } else if (fsa_actions & A_PE_INVOKE) {
 411             do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
 412         } else if (fsa_actions & A_TE_INVOKE) {
 413             do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
 414 
 415             /* Shutdown actions */
 416         } else if (fsa_actions & A_DC_RELEASED) {
 417             do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
 418         } else if (fsa_actions & A_PE_STOP) {
 419             do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
 420         } else if (fsa_actions & A_TE_STOP) {
 421             do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
 422         } else if (fsa_actions & A_SHUTDOWN) {
 423             do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
 424         } else if (fsa_actions & A_LRM_DISCONNECT) {
 425             do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
 426         } else if (fsa_actions & A_HA_DISCONNECT) {
 427             do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
 428         } else if (fsa_actions & A_CIB_STOP) {
 429             do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
 430         } else if (fsa_actions & A_STOP) {
 431             do_fsa_action(fsa_data, A_STOP, do_stop);
 432 
 433             /* exit gracefully */
 434         } else if (fsa_actions & A_EXIT_0) {
 435             do_fsa_action(fsa_data, A_EXIT_0, do_exit);
 436 
 437             /* Error checking and reporting */
 438         } else {
 439             crm_err("Action %s not supported "CRM_XS" 0x%llx",
 440                     fsa_action2string(fsa_actions),
 441                     (unsigned long long) fsa_actions);
 442             register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
 443                                    __func__);
 444         }
 445     }
 446 }
 447 
 448 void
 449 log_fsa_input(fsa_data_t * stored_msg)
     /*  */
 450 {
 451     CRM_ASSERT(stored_msg);
 452     crm_trace("Processing queued input %d", stored_msg->id);
 453     if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
 454         crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
 455 
 456     } else if (stored_msg->data == NULL) {
 457         crm_trace("FSA processing input from %s", stored_msg->origin);
 458 
 459     } else {
 460         ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
 461                                                       __func__);
 462 
 463         crm_trace("FSA processing XML message from %s", stored_msg->origin);
 464         crm_log_xml_trace(ha_input->xml, "FSA message data");
 465     }
 466 }
 467 
 468 static void
 469 check_join_counts(fsa_data_t *msg_data)
     /*  */
 470 {
 471     int count;
 472     guint npeers;
 473 
 474     count = crmd_join_phase_count(crm_join_finalized);
 475     if (count > 0) {
 476         crm_err("%d cluster node%s failed to confirm join",
 477                 count, pcmk__plural_s(count));
 478         crmd_join_phase_log(LOG_NOTICE);
 479         return;
 480     }
 481 
 482     npeers = crm_active_peers();
 483     count = crmd_join_phase_count(crm_join_confirmed);
 484     if (count == npeers) {
 485         if (npeers == 1) {
 486             crm_debug("Sole active cluster node is fully joined");
 487         } else {
 488             crm_debug("All %d active cluster nodes are fully joined", count);
 489         }
 490 
 491     } else if (count > npeers) {
 492         crm_err("New election needed because more nodes confirmed join "
 493                 "than are in membership (%d > %u)", count, npeers);
 494         register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 495 
 496     } else if (saved_ccm_membership_id != crm_peer_seq) {
 497         crm_info("New join needed because membership changed (%llu -> %llu)",
 498                  saved_ccm_membership_id, crm_peer_seq);
 499         register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
 500 
 501     } else {
 502         crm_warn("Only %d of %u active cluster nodes fully joined "
 503                  "(%d did not respond to offer)",
 504                  count, npeers, crmd_join_phase_count(crm_join_welcomed));
 505     }
 506 }
 507 
 508 static void
 509 do_state_transition(enum crmd_fsa_state cur_state,
     /*  */
 510                     enum crmd_fsa_state next_state, fsa_data_t *msg_data)
 511 {
 512     int level = LOG_INFO;
 513     int count = 0;
 514     gboolean clear_recovery_bit = TRUE;
 515 #if 0
 516     uint64_t original_fsa_actions = fsa_actions;
 517 #endif
 518 
 519     enum crmd_fsa_cause cause = msg_data->fsa_cause;
 520     enum crmd_fsa_input current_input = msg_data->fsa_input;
 521 
 522     const char *state_from = fsa_state2string(cur_state);
 523     const char *state_to = fsa_state2string(next_state);
 524     const char *input = fsa_input2string(current_input);
 525 
 526     CRM_LOG_ASSERT(cur_state != next_state);
 527 
 528     do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
 529                state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
 530 
 531     if (cur_state == S_IDLE || next_state == S_IDLE) {
 532         level = LOG_NOTICE;
 533     } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
 534         level = LOG_NOTICE;
 535     } else if (cur_state == S_ELECTION) {
 536         level = LOG_NOTICE;
 537     } else if (cur_state == S_STARTING) {
 538         level = LOG_NOTICE;
 539     } else if (next_state == S_RECOVERY) {
 540         level = LOG_WARNING;
 541     }
 542 
 543     do_crm_log(level, "State transition %s -> %s "
 544                CRM_XS " input=%s cause=%s origin=%s",
 545                state_from, state_to, input, fsa_cause2string(cause),
 546                msg_data->origin);
 547 
 548     if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
 549         controld_stop_election_timer();
 550     }
 551 #if 0
 552     if ((fsa_input_register & R_SHUTDOWN)) {
 553         controld_set_fsa_action_flags(A_DC_TIMER_STOP);
 554     }
 555 #endif
 556     if (next_state == S_INTEGRATION) {
 557         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
 558     } else {
 559         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
 560     }
 561 
 562     if (next_state == S_FINALIZE_JOIN) {
 563         controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
 564     } else {
 565         controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
 566     }
 567 
 568     if (next_state != S_PENDING) {
 569         controld_set_fsa_action_flags(A_DC_TIMER_STOP);
 570     }
 571     if (next_state != S_ELECTION) {
 572         highest_born_on = 0;
 573     }
 574     if (next_state != S_IDLE) {
 575         controld_stop_timer(recheck_timer);
 576     }
 577 
 578     if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
 579         populate_cib_nodes(node_update_quick|node_update_all, __func__);
 580     }
 581 
 582     switch (next_state) {
 583         case S_PENDING:
 584             fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local);
 585             /* fall through */
 586         case S_ELECTION:
 587             crm_trace("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state));
 588             update_dc(NULL);
 589             break;
 590         case S_NOT_DC:
 591             election_trigger->counter = 0;
 592             purge_stonith_cleanup();
 593 
 594             if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
 595                 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
 596                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 597             }
 598             CRM_LOG_ASSERT(fsa_our_dc != NULL);
 599             if (fsa_our_dc == NULL) {
 600                 crm_err("Reached S_NOT_DC without a DC" " being recorded");
 601             }
 602             break;
 603         case S_RECOVERY:
 604             clear_recovery_bit = FALSE;
 605             break;
 606 
 607         case S_FINALIZE_JOIN:
 608             CRM_LOG_ASSERT(AM_I_DC);
 609             if (cause == C_TIMER_POPPED) {
 610                 crm_warn("Progressed to state %s after %s",
 611                          fsa_state2string(next_state), fsa_cause2string(cause));
 612             }
 613             count = crmd_join_phase_count(crm_join_welcomed);
 614             if (count > 0) {
 615                 crm_warn("%d cluster node%s failed to respond to join offer",
 616                          count, pcmk__plural_s(count));
 617                 crmd_join_phase_log(LOG_NOTICE);
 618 
 619             } else {
 620                 crm_debug("All cluster nodes (%d) responded to join offer",
 621                           crmd_join_phase_count(crm_join_integrated));
 622             }
 623             break;
 624 
 625         case S_POLICY_ENGINE:
 626             election_trigger->counter = 0;
 627             CRM_LOG_ASSERT(AM_I_DC);
 628             if (cause == C_TIMER_POPPED) {
 629                 crm_info("Progressed to state %s after %s",
 630                          fsa_state2string(next_state), fsa_cause2string(cause));
 631             }
 632             check_join_counts(msg_data);
 633             break;
 634 
 635         case S_STOPPING:
 636         case S_TERMINATE:
 637             /* possibly redundant */
 638             controld_set_fsa_input_flags(R_SHUTDOWN);
 639             break;
 640 
 641         case S_IDLE:
 642             CRM_LOG_ASSERT(AM_I_DC);
 643             if (pcmk_is_set(fsa_input_register, R_SHUTDOWN)) {
 644                 crm_info("(Re)Issuing shutdown request now" " that we are the DC");
 645                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 646             }
 647             controld_start_recheck_timer();
 648             break;
 649 
 650         default:
 651             break;
 652     }
 653 
 654     if (clear_recovery_bit && next_state != S_PENDING) {
 655         controld_clear_fsa_action_flags(A_RECOVER);
 656     } else if (clear_recovery_bit == FALSE) {
 657         controld_set_fsa_action_flags(A_RECOVER);
 658     }
 659 
 660 #if 0
 661     if (original_fsa_actions != fsa_actions) {
 662         fsa_dump_actions(original_fsa_actions ^ fsa_actions, "New actions");
 663     }
 664 #endif
 665 }
/* */
root/daemons/controld/controld_fsa.c

DEFINITIONS
