daemons/controld/controld

/* */
This source file includes following definitions.
init_dotfile
do_fsa_action
do_log
controld_init_fsa_trigger
controld_destroy_fsa_trigger
controld_trigger_fsa_as
s_crmd_fsa
s_crmd_fsa_actions
log_fsa_input
check_join_counts
do_state_transition
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 #include <stdio.h>
  14 #include <stdint.h>                 // uint64_t
  15 #include <string.h>
  16 #include <time.h>
  17 
  18 #include <crm/crm.h>
  19 #include <crm/lrmd.h>
  20 #include <crm/cib.h>
  21 #include <crm/common/xml.h>
  22 #include <crm/cluster/election_internal.h>
  23 #include <crm/cluster.h>
  24 
  25 #include <pacemaker-controld.h>
  26 
  27 //! Triggers an FSA invocation
  28 static crm_trigger_t *fsa_trigger = NULL;
  29 
  30 #define DOT_PREFIX "actions:trace: "
  31 #define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
  32 
  33 static void do_state_transition(enum crmd_fsa_state cur_state,
  34                                 enum crmd_fsa_state next_state,
  35                                 fsa_data_t *msg_data);
  36 
  37 void s_crmd_fsa_actions(fsa_data_t * fsa_data);
  38 void log_fsa_input(fsa_data_t * stored_msg);
  39 void init_dotfile(void);
  40 
  41 void
  42 init_dotfile(void)
     /*  */
  43 {
  44     do_dot_log(DOT_PREFIX "digraph \"g\" {");
  45     do_dot_log(DOT_PREFIX "     size = \"30,30\"");
  46     do_dot_log(DOT_PREFIX "     graph [");
  47     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  48     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  49     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  50     do_dot_log(DOT_PREFIX "             bb = \"0,0,398.922306,478.927856\"");
  51     do_dot_log(DOT_PREFIX "             color = \"black\"");
  52     do_dot_log(DOT_PREFIX "     ]");
  53     do_dot_log(DOT_PREFIX "     node [");
  54     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  55     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  56     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  57     do_dot_log(DOT_PREFIX "             shape = \"ellipse\"");
  58     do_dot_log(DOT_PREFIX "             color = \"black\"");
  59     do_dot_log(DOT_PREFIX "     ]");
  60     do_dot_log(DOT_PREFIX "     edge [");
  61     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  62     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  63     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  64     do_dot_log(DOT_PREFIX "             color = \"black\"");
  65     do_dot_log(DOT_PREFIX "     ]");
  66     do_dot_log(DOT_PREFIX "// special nodes");
  67     do_dot_log(DOT_PREFIX "     \"S_PENDING\" ");
  68     do_dot_log(DOT_PREFIX "     [");
  69     do_dot_log(DOT_PREFIX "      color = \"blue\"");
  70     do_dot_log(DOT_PREFIX "      fontcolor = \"blue\"");
  71     do_dot_log(DOT_PREFIX "      ]");
  72     do_dot_log(DOT_PREFIX "     \"S_TERMINATE\" ");
  73     do_dot_log(DOT_PREFIX "     [");
  74     do_dot_log(DOT_PREFIX "      color = \"red\"");
  75     do_dot_log(DOT_PREFIX "      fontcolor = \"red\"");
  76     do_dot_log(DOT_PREFIX "      ]");
  77     do_dot_log(DOT_PREFIX "// DC only nodes");
  78     do_dot_log(DOT_PREFIX "     \"S_INTEGRATION\" [ fontcolor = \"green\" ]");
  79     do_dot_log(DOT_PREFIX "     \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
  80     do_dot_log(DOT_PREFIX "     \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
  81     do_dot_log(DOT_PREFIX "     \"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
  82     do_dot_log(DOT_PREFIX "     \"S_IDLE\" [ fontcolor = \"green\" ]");
  83 }
  84 
  85 static void
  86 do_fsa_action(fsa_data_t * fsa_data, long long an_action,
     /*  */
  87               void (*function) (long long action,
  88                                 enum crmd_fsa_cause cause,
  89                                 enum crmd_fsa_state cur_state,
  90                                 enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
  91 {
  92     controld_clear_fsa_action_flags(an_action);
  93     crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
  94     function(an_action, fsa_data->fsa_cause, controld_globals.fsa_state,
  95              fsa_data->fsa_input, fsa_data);
  96 }
  97 
  98 static const uint64_t startup_actions =
  99     A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_HA_CONNECT | A_READCONFIG |
 100     A_STARTED | A_CL_JOIN_QUERY;
 101 
 102 // A_LOG, A_WARN, A_ERROR
 103 void
 104 do_log(long long action, enum crmd_fsa_cause cause,
     /*  */
 105        enum crmd_fsa_state cur_state,
 106        enum crmd_fsa_input current_input, fsa_data_t *msg_data)
 107 {
 108     unsigned log_type = LOG_TRACE;
 109 
 110     if (action & A_LOG) {
 111         log_type = LOG_INFO;
 112     } else if (action & A_WARN) {
 113         log_type = LOG_WARNING;
 114     } else if (action & A_ERROR) {
 115         log_type = LOG_ERR;
 116     }
 117 
 118     do_crm_log(log_type, "Input %s received in state %s from %s",
 119                fsa_input2string(msg_data->fsa_input),
 120                fsa_state2string(cur_state), msg_data->origin);
 121 
 122     if (msg_data->data_type == fsa_dt_ha_msg) {
 123         ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
 124 
 125         crm_log_xml_debug(input->msg, __func__);
 126 
 127     } else if (msg_data->data_type == fsa_dt_xml) {
 128         xmlNode *input = fsa_typed_data(msg_data->data_type);
 129 
 130         crm_log_xml_debug(input, __func__);
 131 
 132     } else if (msg_data->data_type == fsa_dt_lrm) {
 133         lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
 134 
 135         do_crm_log(log_type,
 136                    "Resource %s: Call ID %d returned %d (%d)."
 137                    "  New status if rc=0: %s",
 138                    input->rsc_id, input->call_id, input->rc,
 139                    input->op_status, (char *)input->user_data);
 140     }
 141 }
 142 
 143 /*!
 144  * \internal
 145  * \brief Initialize the FSA trigger
 146  */
 147 void
 148 controld_init_fsa_trigger(void)
     /*  */
 149 {
 150     fsa_trigger = mainloop_add_trigger(G_PRIORITY_HIGH, crm_fsa_trigger, NULL);
 151 }
 152 
 153 /*!
 154  * \internal
 155  * \brief Destroy the FSA trigger
 156  */
 157 void
 158 controld_destroy_fsa_trigger(void)
     /*  */
 159 {
 160     // This basically will not work, since mainloop has a reference to it
 161     mainloop_destroy_trigger(fsa_trigger);
 162     fsa_trigger = NULL;
 163 }
 164 
 165 /*!
 166  * \internal
 167  * \brief Trigger an FSA invocation
 168  *
 169  * \param[in] fn    Calling function name
 170  * \param[in] line  Line number where call occurred
 171  */
 172 void
 173 controld_trigger_fsa_as(const char *fn, int line)
     /*  */
 174 {
 175     if (fsa_trigger != NULL) {
 176         crm_trace("%s:%d - Triggered FSA invocation", fn, line);
 177         mainloop_set_trigger(fsa_trigger);
 178     }
 179 }
 180 
 181 enum crmd_fsa_state
 182 s_crmd_fsa(enum crmd_fsa_cause cause)
     /*  */
 183 {
 184     controld_globals_t *globals = &controld_globals;
 185     fsa_data_t *fsa_data = NULL;
 186     uint64_t register_copy = controld_globals.fsa_input_register;
 187     uint64_t new_actions = A_NOTHING;
 188     enum crmd_fsa_state last_state;
 189 
 190     crm_trace("FSA invoked with Cause: %s\tState: %s",
 191               fsa_cause2string(cause),
 192               fsa_state2string(globals->fsa_state));
 193 
 194     fsa_dump_actions(controld_globals.fsa_actions, "Initial");
 195 
 196     controld_clear_global_flags(controld_fsa_is_stalled);
 197     if ((controld_globals.fsa_message_queue == NULL)
 198         && (controld_globals.fsa_actions != A_NOTHING)) {
 199         /* fake the first message so we can get into the loop */
 200         fsa_data = pcmk__assert_alloc(1, sizeof(fsa_data_t));
 201         fsa_data->fsa_input = I_NULL;
 202         fsa_data->fsa_cause = C_FSA_INTERNAL;
 203         fsa_data->origin = __func__;
 204         fsa_data->data_type = fsa_dt_none;
 205         controld_globals.fsa_message_queue
 206             = g_list_append(controld_globals.fsa_message_queue, fsa_data);
 207     }
 208     while ((controld_globals.fsa_message_queue != NULL)
 209            && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 210         crm_trace("Checking messages (%d remaining)",
 211                   g_list_length(controld_globals.fsa_message_queue));
 212 
 213         fsa_data = get_message();
 214         if(fsa_data == NULL) {
 215             continue;
 216         }
 217 
 218         log_fsa_input(fsa_data);
 219 
 220         /* add any actions back to the queue */
 221         controld_set_fsa_action_flags(fsa_data->actions);
 222         fsa_dump_actions(fsa_data->actions, "Restored actions");
 223 
 224         /* get the next batch of actions */
 225         new_actions = controld_fsa_get_action(fsa_data->fsa_input);
 226         controld_set_fsa_action_flags(new_actions);
 227         fsa_dump_actions(new_actions, "New actions");
 228 
 229         if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
 230             crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
 231                       fsa_input2string(fsa_data->fsa_input),
 232                       fsa_state2string(globals->fsa_state),
 233                       fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
 234         }
 235 
 236         /* logging : *before* the state is changed */
 237         if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
 238             do_fsa_action(fsa_data, A_ERROR, do_log);
 239         }
 240         if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
 241             do_fsa_action(fsa_data, A_WARN, do_log);
 242         }
 243         if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
 244             do_fsa_action(fsa_data, A_LOG, do_log);
 245         }
 246 
 247         /* update state variables */
 248         last_state = globals->fsa_state;
 249         globals->fsa_state = controld_fsa_get_next_state(fsa_data->fsa_input);
 250 
 251         /*
 252          * Remove certain actions during shutdown
 253          */
 254         if ((globals->fsa_state == S_STOPPING)
 255             || pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 256             controld_clear_fsa_action_flags(startup_actions);
 257         }
 258 
 259         /*
 260          * Hook for change of state.
 261          * Allows actions to be added or removed when entering a state
 262          */
 263         if (last_state != globals->fsa_state) {
 264             do_state_transition(last_state, globals->fsa_state, fsa_data);
 265         } else {
 266             do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
 267                        " \tInput=%s \tOrigin=%s() \tid=%d",
 268                        fsa_state2string(globals->fsa_state),
 269                        fsa_cause2string(fsa_data->fsa_cause),
 270                        fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
 271         }
 272 
 273         /* start doing things... */
 274         s_crmd_fsa_actions(fsa_data);
 275         delete_fsa_input(fsa_data);
 276     }
 277 
 278     if ((controld_globals.fsa_message_queue != NULL)
 279         || (controld_globals.fsa_actions != A_NOTHING)
 280         || pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 281 
 282         crm_debug("Exiting the FSA: queue=%d, fsa_actions=%#llx, stalled=%s",
 283                   g_list_length(controld_globals.fsa_message_queue),
 284                   (unsigned long long) controld_globals.fsa_actions,
 285                   pcmk__flag_text(controld_globals.flags,
 286                                   controld_fsa_is_stalled));
 287     } else {
 288         crm_trace("Exiting the FSA");
 289     }
 290 
 291     /* cleanup inputs? */
 292     if (register_copy != controld_globals.fsa_input_register) {
 293         uint64_t same = register_copy & controld_globals.fsa_input_register;
 294 
 295         fsa_dump_inputs(LOG_DEBUG, "Added",
 296                         controld_globals.fsa_input_register ^ same);
 297         fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
 298     }
 299 
 300     fsa_dump_actions(controld_globals.fsa_actions, "Remaining");
 301     fsa_dump_queue(LOG_DEBUG);
 302 
 303     return globals->fsa_state;
 304 }
 305 
 306 void
 307 s_crmd_fsa_actions(fsa_data_t * fsa_data)
     /*  */
 308 {
 309     /*
 310      * Process actions in order of priority but do only one
 311      * action at a time to avoid complicating the ordering.
 312      */
 313     CRM_CHECK(fsa_data != NULL, return);
 314     while ((controld_globals.fsa_actions != A_NOTHING)
 315            && !pcmk_is_set(controld_globals.flags, controld_fsa_is_stalled)) {
 316 
 317         /* regular action processing in order of action priority
 318          *
 319          * Make sure all actions that connect to required systems
 320          * are performed first
 321          */
 322         if (pcmk_is_set(controld_globals.fsa_actions, A_ERROR)) {
 323             do_fsa_action(fsa_data, A_ERROR, do_log);
 324         } else if (pcmk_is_set(controld_globals.fsa_actions, A_WARN)) {
 325             do_fsa_action(fsa_data, A_WARN, do_log);
 326         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LOG)) {
 327             do_fsa_action(fsa_data, A_LOG, do_log);
 328 
 329             /* get out of here NOW! before anything worse happens */
 330         } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_1)) {
 331             do_fsa_action(fsa_data, A_EXIT_1, do_exit);
 332 
 333             /* sub-system restart */
 334         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 335                                       O_LRM_RECONNECT)) {
 336             do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
 337 
 338         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 339                                       O_CIB_RESTART)) {
 340             do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
 341 
 342         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 343                                       O_PE_RESTART)) {
 344             do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
 345 
 346         } else if (pcmk_all_flags_set(controld_globals.fsa_actions,
 347                                       O_TE_RESTART)) {
 348             do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
 349 
 350             /* essential start tasks */
 351         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTUP)) {
 352             do_fsa_action(fsa_data, A_STARTUP, do_startup);
 353         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_START)) {
 354             do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
 355         } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_CONNECT)) {
 356             do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
 357         } else if (pcmk_is_set(controld_globals.fsa_actions, A_READCONFIG)) {
 358             do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
 359 
 360             /* sub-system start/connect */
 361         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_CONNECT)) {
 362             do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
 363         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_START)) {
 364             do_fsa_action(fsa_data, A_TE_START, do_te_control);
 365         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_START)) {
 366             do_fsa_action(fsa_data, A_PE_START, do_pe_control);
 367 
 368             /* Timers */
 369         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TIMER_STOP)) {
 370             do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
 371         } else if (pcmk_is_set(controld_globals.fsa_actions,
 372                                A_INTEGRATE_TIMER_STOP)) {
 373             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
 374         } else if (pcmk_is_set(controld_globals.fsa_actions,
 375                                A_INTEGRATE_TIMER_START)) {
 376             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
 377         } else if (pcmk_is_set(controld_globals.fsa_actions,
 378                                A_FINALIZE_TIMER_STOP)) {
 379             do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
 380         } else if (pcmk_is_set(controld_globals.fsa_actions,
 381                                A_FINALIZE_TIMER_START)) {
 382             do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
 383 
 384             /*
 385              * Highest priority actions
 386              */
 387         } else if (pcmk_is_set(controld_globals.fsa_actions, A_MSG_ROUTE)) {
 388             do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
 389         } else if (pcmk_is_set(controld_globals.fsa_actions, A_RECOVER)) {
 390             do_fsa_action(fsa_data, A_RECOVER, do_recover);
 391         } else if (pcmk_is_set(controld_globals.fsa_actions,
 392                                A_CL_JOIN_RESULT)) {
 393             do_fsa_action(fsa_data, A_CL_JOIN_RESULT,
 394                           do_cl_join_finalize_respond);
 395 
 396         } else if (pcmk_is_set(controld_globals.fsa_actions,
 397                                A_CL_JOIN_REQUEST)) {
 398             do_fsa_action(fsa_data, A_CL_JOIN_REQUEST,
 399                           do_cl_join_offer_respond);
 400 
 401         } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN_REQ)) {
 402             do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
 403         } else if (pcmk_is_set(controld_globals.fsa_actions, A_ELECTION_VOTE)) {
 404             do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
 405         } else if (pcmk_is_set(controld_globals.fsa_actions,
 406                                A_ELECTION_COUNT)) {
 407             do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
 408 
 409         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_EVENT)) {
 410             do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
 411 
 412             /*
 413              * High priority actions
 414              */
 415         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STARTED)) {
 416             do_fsa_action(fsa_data, A_STARTED, do_started);
 417         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CL_JOIN_QUERY)) {
 418             do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
 419         } else if (pcmk_is_set(controld_globals.fsa_actions,
 420                                A_DC_TIMER_START)) {
 421             do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
 422 
 423             /*
 424              * Medium priority actions
 425              * - Membership
 426              */
 427         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_TAKEOVER)) {
 428             do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
 429         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASE)) {
 430             do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
 431         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_JOIN_FINAL)) {
 432             do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
 433         } else if (pcmk_is_set(controld_globals.fsa_actions,
 434                                A_ELECTION_CHECK)) {
 435             do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
 436 
 437         } else if (pcmk_is_set(controld_globals.fsa_actions,
 438                                A_ELECTION_START)) {
 439             do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
 440 
 441         } else if (pcmk_is_set(controld_globals.fsa_actions,
 442                                A_DC_JOIN_OFFER_ALL)) {
 443             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
 444 
 445         } else if (pcmk_is_set(controld_globals.fsa_actions,
 446                                A_DC_JOIN_OFFER_ONE)) {
 447             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
 448 
 449         } else if (pcmk_is_set(controld_globals.fsa_actions,
 450                                A_DC_JOIN_PROCESS_REQ)) {
 451             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ,
 452                           do_dc_join_filter_offer);
 453 
 454         } else if (pcmk_is_set(controld_globals.fsa_actions,
 455                                A_DC_JOIN_PROCESS_ACK)) {
 456             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
 457 
 458         } else if (pcmk_is_set(controld_globals.fsa_actions,
 459                                A_DC_JOIN_FINALIZE)) {
 460             do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
 461 
 462         } else if (pcmk_is_set(controld_globals.fsa_actions,
 463                                A_CL_JOIN_ANNOUNCE)) {
 464             do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
 465 
 466             /*
 467              * Low(er) priority actions
 468              * Make sure the CIB is always updated before invoking the
 469              * scheduler, and the scheduler before the transition engine.
 470              */
 471         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_HALT)) {
 472             do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
 473         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_CANCEL)) {
 474             do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
 475         } else if (pcmk_is_set(controld_globals.fsa_actions, A_LRM_INVOKE)) {
 476             do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
 477         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_INVOKE)) {
 478             do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
 479         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_INVOKE)) {
 480             do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
 481 
 482             /* Shutdown actions */
 483         } else if (pcmk_is_set(controld_globals.fsa_actions, A_DC_RELEASED)) {
 484             do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
 485         } else if (pcmk_is_set(controld_globals.fsa_actions, A_PE_STOP)) {
 486             do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
 487         } else if (pcmk_is_set(controld_globals.fsa_actions, A_TE_STOP)) {
 488             do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
 489         } else if (pcmk_is_set(controld_globals.fsa_actions, A_SHUTDOWN)) {
 490             do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
 491         } else if (pcmk_is_set(controld_globals.fsa_actions,
 492                                A_LRM_DISCONNECT)) {
 493             do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
 494 
 495         } else if (pcmk_is_set(controld_globals.fsa_actions, A_HA_DISCONNECT)) {
 496             do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
 497         } else if (pcmk_is_set(controld_globals.fsa_actions, A_CIB_STOP)) {
 498             do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
 499         } else if (pcmk_is_set(controld_globals.fsa_actions, A_STOP)) {
 500             do_fsa_action(fsa_data, A_STOP, do_stop);
 501 
 502             /* exit gracefully */
 503         } else if (pcmk_is_set(controld_globals.fsa_actions, A_EXIT_0)) {
 504             do_fsa_action(fsa_data, A_EXIT_0, do_exit);
 505 
 506             /* Error checking and reporting */
 507         } else {
 508             crm_err("Action %s not supported "CRM_XS" %#llx",
 509                     fsa_action2string(controld_globals.fsa_actions),
 510                     (unsigned long long) controld_globals.fsa_actions);
 511             register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL,
 512                                    __func__);
 513         }
 514     }
 515 }
 516 
 517 void
 518 log_fsa_input(fsa_data_t * stored_msg)
     /*  */
 519 {
 520     CRM_ASSERT(stored_msg);
 521     crm_trace("Processing queued input %d", stored_msg->id);
 522     if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
 523         crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
 524 
 525     } else if (stored_msg->data == NULL) {
 526         crm_trace("FSA processing input from %s", stored_msg->origin);
 527 
 528     } else {
 529         ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg,
 530                                                       __func__);
 531 
 532         crm_trace("FSA processing XML message from %s", stored_msg->origin);
 533         crm_log_xml_trace(ha_input->xml, "FSA message data");
 534     }
 535 }
 536 
 537 static void
 538 check_join_counts(fsa_data_t *msg_data)
     /*  */
 539 {
 540     int count;
 541     guint npeers;
 542 
 543     count = crmd_join_phase_count(crm_join_finalized);
 544     if (count > 0) {
 545         crm_err("%d cluster node%s failed to confirm join",
 546                 count, pcmk__plural_s(count));
 547         crmd_join_phase_log(LOG_NOTICE);
 548         return;
 549     }
 550 
 551     npeers = pcmk__cluster_num_active_nodes();
 552     count = crmd_join_phase_count(crm_join_confirmed);
 553     if (count == npeers) {
 554         if (npeers == 1) {
 555             crm_debug("Sole active cluster node is fully joined");
 556         } else {
 557             crm_debug("All %d active cluster nodes are fully joined", count);
 558         }
 559 
 560     } else if (count > npeers) {
 561         crm_err("New election needed because more nodes confirmed join "
 562                 "than are in membership (%d > %u)", count, npeers);
 563         register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 564 
 565     } else if (controld_globals.membership_id != crm_peer_seq) {
 566         crm_info("New join needed because membership changed (%llu -> %llu)",
 567                  controld_globals.membership_id, crm_peer_seq);
 568         register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
 569 
 570     } else {
 571         crm_warn("Only %d of %u active cluster nodes fully joined "
 572                  "(%d did not respond to offer)",
 573                  count, npeers, crmd_join_phase_count(crm_join_welcomed));
 574     }
 575 }
 576 
 577 static void
 578 do_state_transition(enum crmd_fsa_state cur_state,
     /*  */
 579                     enum crmd_fsa_state next_state, fsa_data_t *msg_data)
 580 {
 581     int level = LOG_INFO;
 582     int count = 0;
 583     gboolean clear_recovery_bit = TRUE;
 584 #if 0
 585     uint64_t original_fsa_actions = controld_globals.fsa_actions;
 586 #endif
 587 
 588     enum crmd_fsa_cause cause = msg_data->fsa_cause;
 589     enum crmd_fsa_input current_input = msg_data->fsa_input;
 590 
 591     const char *state_from = fsa_state2string(cur_state);
 592     const char *state_to = fsa_state2string(next_state);
 593     const char *input = fsa_input2string(current_input);
 594 
 595     CRM_LOG_ASSERT(cur_state != next_state);
 596 
 597     do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
 598                state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
 599 
 600     if (cur_state == S_IDLE || next_state == S_IDLE) {
 601         level = LOG_NOTICE;
 602     } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
 603         level = LOG_NOTICE;
 604     } else if (cur_state == S_ELECTION) {
 605         level = LOG_NOTICE;
 606     } else if (cur_state == S_STARTING) {
 607         level = LOG_NOTICE;
 608     } else if (next_state == S_RECOVERY) {
 609         level = LOG_WARNING;
 610     }
 611 
 612     do_crm_log(level, "State transition %s -> %s "
 613                CRM_XS " input=%s cause=%s origin=%s",
 614                state_from, state_to, input, fsa_cause2string(cause),
 615                msg_data->origin);
 616 
 617     if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
 618         controld_stop_current_election_timeout();
 619     }
 620     if (next_state == S_INTEGRATION) {
 621         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_START);
 622     } else {
 623         controld_set_fsa_action_flags(A_INTEGRATE_TIMER_STOP);
 624     }
 625 
 626     if (next_state == S_FINALIZE_JOIN) {
 627         controld_set_fsa_action_flags(A_FINALIZE_TIMER_START);
 628     } else {
 629         controld_set_fsa_action_flags(A_FINALIZE_TIMER_STOP);
 630     }
 631 
 632     if (next_state != S_PENDING) {
 633         controld_set_fsa_action_flags(A_DC_TIMER_STOP);
 634     }
 635     if (next_state != S_IDLE) {
 636         controld_stop_recheck_timer();
 637     }
 638 
 639     if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
 640         populate_cib_nodes(node_update_quick|node_update_all, __func__);
 641     }
 642 
 643     switch (next_state) {
 644         case S_PENDING:
 645             {
 646                 cib_t *cib_conn = controld_globals.cib_conn;
 647                 cib_conn->cmds->set_secondary(cib_conn, cib_scope_local);
 648             }
 649             update_dc(NULL);
 650             break;
 651 
 652         case S_ELECTION:
 653             update_dc(NULL);
 654             break;
 655 
 656         case S_NOT_DC:
 657             controld_reset_counter_election_timer();
 658             purge_stonith_cleanup();
 659 
 660             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 661                 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
 662                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 663             }
 664             CRM_LOG_ASSERT(controld_globals.dc_name != NULL);
 665             if (controld_globals.dc_name == NULL) {
 666                 crm_err("Reached S_NOT_DC without a DC" " being recorded");
 667             }
 668             break;
 669 
 670         case S_RECOVERY:
 671             clear_recovery_bit = FALSE;
 672             break;
 673 
 674         case S_FINALIZE_JOIN:
 675             CRM_LOG_ASSERT(AM_I_DC);
 676             if (cause == C_TIMER_POPPED) {
 677                 crm_warn("Progressed to state %s after %s",
 678                          fsa_state2string(next_state), fsa_cause2string(cause));
 679             }
 680             count = crmd_join_phase_count(crm_join_welcomed);
 681             if (count > 0) {
 682                 crm_warn("%d cluster node%s failed to respond to join offer",
 683                          count, pcmk__plural_s(count));
 684                 crmd_join_phase_log(LOG_NOTICE);
 685 
 686             } else {
 687                 crm_debug("All cluster nodes (%d) responded to join offer",
 688                           crmd_join_phase_count(crm_join_integrated));
 689             }
 690             break;
 691 
 692         case S_POLICY_ENGINE:
 693             controld_reset_counter_election_timer();
 694             CRM_LOG_ASSERT(AM_I_DC);
 695             if (cause == C_TIMER_POPPED) {
 696                 crm_info("Progressed to state %s after %s",
 697                          fsa_state2string(next_state), fsa_cause2string(cause));
 698             }
 699             check_join_counts(msg_data);
 700             break;
 701 
 702         case S_STOPPING:
 703         case S_TERMINATE:
 704             /* possibly redundant */
 705             controld_set_fsa_input_flags(R_SHUTDOWN);
 706             break;
 707 
 708         case S_IDLE:
 709             CRM_LOG_ASSERT(AM_I_DC);
 710             if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 711                 crm_info("(Re)Issuing shutdown request now" " that we are the DC");
 712                 controld_set_fsa_action_flags(A_SHUTDOWN_REQ);
 713             }
 714             controld_start_recheck_timer();
 715             break;
 716 
 717         default:
 718             break;
 719     }
 720 
 721     if (clear_recovery_bit && next_state != S_PENDING) {
 722         controld_clear_fsa_action_flags(A_RECOVER);
 723     } else if (clear_recovery_bit == FALSE) {
 724         controld_set_fsa_action_flags(A_RECOVER);
 725     }
 726 
 727 #if 0
 728     if (original_fsa_actions != controld_globals.fsa_actions) {
 729         fsa_dump_actions(original_fsa_actions ^ controld_globals.fsa_actions,
 730                          "New actions");
 731     }
 732 #endif
 733 }
/* */
root/daemons/controld/controld_fsa.c

DEFINITIONS
