crmd/fsa.c

/* */
This source file includes following definitions.
init_dotfile
do_fsa_action
s_crmd_fsa
s_crmd_fsa_actions
log_fsa_input
do_state_transition
dump_rsc_info
ghash_print_node
   1 /*
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <sys/param.h>
  22 #include <stdio.h>
  23 #include <string.h>
  24 #include <time.h>
  25 
  26 #include <crm/crm.h>
  27 #include <crm/lrmd.h>
  28 #include <crm/cib.h>
  29 #include <crm/msg_xml.h>
  30 #include <crm/common/xml.h>
  31 #include <crm/cluster/election.h>
  32 
  33 #include <crm/cluster.h>
  34 
  35 #include <crmd_messages.h>
  36 #include <crmd_fsa.h>
  37 #include <tengine.h>
  38 #include <fsa_proto.h>
  39 #include <fsa_matrix.h>
  40 
  41 char *fsa_our_dc = NULL;
  42 cib_t *fsa_cib_conn = NULL;
  43 char *fsa_our_dc_version = NULL;
  44 
  45 char *fsa_our_uuid = NULL;
  46 char *fsa_our_uname = NULL;
  47 
  48 char *fsa_cluster_name = NULL;
  49 
  50 #if SUPPORT_HEARTBEAT
  51 ll_cluster_t *fsa_cluster_conn;
  52 #endif
  53 
  54 election_t *fsa_election = NULL;
  55 
  56 fsa_timer_t *wait_timer = NULL;        /* How long to wait before retrying to connect to the cib/lrmd/ccm */
  57 fsa_timer_t *recheck_timer = NULL;     /* Periodically re-run the PE to account for time based rules/preferences */
  58 fsa_timer_t *election_trigger = NULL;  /* How long to wait at startup, or after an election, for the DC to make contact */
  59 fsa_timer_t *transition_timer = NULL;  /* How long to delay the start of a new transition with the expectation something else might happen too */
  60 fsa_timer_t *integration_timer = NULL;
  61 fsa_timer_t *finalization_timer = NULL;
  62 fsa_timer_t *shutdown_escalation_timer = NULL; /* How long to wait for the DC to stop all resources and give us the all-clear to shut down */
  63 
  64 volatile gboolean do_fsa_stall = FALSE;
  65 volatile long long fsa_input_register = 0;
  66 volatile long long fsa_actions = A_NOTHING;
  67 volatile enum crmd_fsa_state fsa_state = S_STARTING;
  68 
  69 extern uint highest_born_on;
  70 extern uint num_join_invites;
  71 extern void initialize_join(gboolean before);
  72 
  73 #define DOT_PREFIX "actions:trace: "
  74 #define do_dot_log(fmt, args...)     crm_trace( fmt, ##args)
  75 
  76 long long do_state_transition(long long actions,
  77                               enum crmd_fsa_state cur_state,
  78                               enum crmd_fsa_state next_state, fsa_data_t * msg_data);
  79 
  80 void dump_rsc_info(void);
  81 void dump_rsc_info_callback(const xmlNode * msg, int call_id, int rc,
  82                             xmlNode * output, void *user_data);
  83 
  84 void ghash_print_node(gpointer key, gpointer value, gpointer user_data);
  85 
  86 void s_crmd_fsa_actions(fsa_data_t * fsa_data);
  87 void log_fsa_input(fsa_data_t * stored_msg);
  88 void init_dotfile(void);
  89 
  90 void
  91 init_dotfile(void)
     /*  */
  92 {
  93     do_dot_log(DOT_PREFIX "digraph \"g\" {");
  94     do_dot_log(DOT_PREFIX "     size = \"30,30\"");
  95     do_dot_log(DOT_PREFIX "     graph [");
  96     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
  97     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
  98     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
  99     do_dot_log(DOT_PREFIX "             bb = \"0,0,398.922306,478.927856\"");
 100     do_dot_log(DOT_PREFIX "             color = \"black\"");
 101     do_dot_log(DOT_PREFIX "     ]");
 102     do_dot_log(DOT_PREFIX "     node [");
 103     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
 104     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
 105     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
 106     do_dot_log(DOT_PREFIX "             shape = \"ellipse\"");
 107     do_dot_log(DOT_PREFIX "             color = \"black\"");
 108     do_dot_log(DOT_PREFIX "     ]");
 109     do_dot_log(DOT_PREFIX "     edge [");
 110     do_dot_log(DOT_PREFIX "             fontsize = \"12\"");
 111     do_dot_log(DOT_PREFIX "             fontname = \"Times-Roman\"");
 112     do_dot_log(DOT_PREFIX "             fontcolor = \"black\"");
 113     do_dot_log(DOT_PREFIX "             color = \"black\"");
 114     do_dot_log(DOT_PREFIX "     ]");
 115     do_dot_log(DOT_PREFIX "// special nodes");
 116     do_dot_log(DOT_PREFIX "     \"S_PENDING\" ");
 117     do_dot_log(DOT_PREFIX "     [");
 118     do_dot_log(DOT_PREFIX "      color = \"blue\"");
 119     do_dot_log(DOT_PREFIX "      fontcolor = \"blue\"");
 120     do_dot_log(DOT_PREFIX "      ]");
 121     do_dot_log(DOT_PREFIX "     \"S_TERMINATE\" ");
 122     do_dot_log(DOT_PREFIX "     [");
 123     do_dot_log(DOT_PREFIX "      color = \"red\"");
 124     do_dot_log(DOT_PREFIX "      fontcolor = \"red\"");
 125     do_dot_log(DOT_PREFIX "      ]");
 126     do_dot_log(DOT_PREFIX "// DC only nodes");
 127     do_dot_log(DOT_PREFIX "     \"S_INTEGRATION\" [ fontcolor = \"green\" ]");
 128     do_dot_log(DOT_PREFIX "     \"S_POLICY_ENGINE\" [ fontcolor = \"green\" ]");
 129     do_dot_log(DOT_PREFIX "     \"S_TRANSITION_ENGINE\" [ fontcolor = \"green\" ]");
 130     do_dot_log(DOT_PREFIX "     \"S_RELEASE_DC\" [ fontcolor = \"green\" ]");
 131     do_dot_log(DOT_PREFIX "     \"S_IDLE\" [ fontcolor = \"green\" ]");
 132 }
 133 
 134 static void
 135 do_fsa_action(fsa_data_t * fsa_data, long long an_action,
     /*  */
 136               void (*function) (long long action,
 137                                 enum crmd_fsa_cause cause,
 138                                 enum crmd_fsa_state cur_state,
 139                                 enum crmd_fsa_input cur_input, fsa_data_t * msg_data))
 140 {
 141     fsa_actions &= ~an_action;
 142     crm_trace(DOT_PREFIX "\t// %s", fsa_action2string(an_action));
 143     function(an_action, fsa_data->fsa_cause, fsa_state, fsa_data->fsa_input, fsa_data);
 144 }
 145 
 146 static long long startup_actions =
 147     A_STARTUP | A_CIB_START | A_LRM_CONNECT | A_CCM_CONNECT | A_HA_CONNECT | A_READCONFIG |
 148     A_STARTED | A_CL_JOIN_QUERY;
 149 
 150 enum crmd_fsa_state
 151 s_crmd_fsa(enum crmd_fsa_cause cause)
     /*  */
 152 {
 153     fsa_data_t *fsa_data = NULL;
 154     long long register_copy = fsa_input_register;
 155     long long new_actions = A_NOTHING;
 156     enum crmd_fsa_state last_state;
 157 
 158     crm_trace("FSA invoked with Cause: %s\tState: %s",
 159               fsa_cause2string(cause), fsa_state2string(fsa_state));
 160 
 161     fsa_dump_actions(fsa_actions, "Initial");
 162 
 163     do_fsa_stall = FALSE;
 164     if (is_message() == FALSE && fsa_actions != A_NOTHING) {
 165         /* fake the first message so we can get into the loop */
 166         fsa_data = calloc(1, sizeof(fsa_data_t));
 167         fsa_data->fsa_input = I_NULL;
 168         fsa_data->fsa_cause = C_FSA_INTERNAL;
 169         fsa_data->origin = __FUNCTION__;
 170         fsa_data->data_type = fsa_dt_none;
 171         fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
 172         fsa_data = NULL;
 173     }
 174     while (is_message() && do_fsa_stall == FALSE) {
 175         crm_trace("Checking messages (%d remaining)", g_list_length(fsa_message_queue));
 176 
 177         fsa_data = get_message();
 178         if(fsa_data == NULL) {
 179             continue;
 180         }
 181 
 182         log_fsa_input(fsa_data);
 183 
 184         /* add any actions back to the queue */
 185         fsa_actions |= fsa_data->actions;
 186         fsa_dump_actions(fsa_data->actions, "Restored actions");
 187 
 188         /* get the next batch of actions */
 189         new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state];
 190         fsa_actions |= new_actions;
 191         fsa_dump_actions(new_actions, "New actions");
 192 
 193         if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
 194             crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
 195                       fsa_input2string(fsa_data->fsa_input),
 196                       fsa_state2string(fsa_state),
 197                       fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
 198         }
 199 
 200         /* logging : *before* the state is changed */
 201         if (is_set(fsa_actions, A_ERROR)) {
 202             do_fsa_action(fsa_data, A_ERROR, do_log);
 203         }
 204         if (is_set(fsa_actions, A_WARN)) {
 205             do_fsa_action(fsa_data, A_WARN, do_log);
 206         }
 207         if (is_set(fsa_actions, A_LOG)) {
 208             do_fsa_action(fsa_data, A_LOG, do_log);
 209         }
 210 
 211         /* update state variables */
 212         last_state = fsa_state;
 213         fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state];
 214 
 215         /*
 216          * Remove certain actions during shutdown
 217          */
 218         if (fsa_state == S_STOPPING || ((fsa_input_register & R_SHUTDOWN) == R_SHUTDOWN)) {
 219             clear_bit(fsa_actions, startup_actions);
 220         }
 221 
 222         /*
 223          * Hook for change of state.
 224          * Allows actions to be added or removed when entering a state
 225          */
 226         if (last_state != fsa_state) {
 227             fsa_actions = do_state_transition(fsa_actions, last_state, fsa_state, fsa_data);
 228         } else {
 229             do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
 230                        " \tInput=%s \tOrigin=%s() \tid=%d",
 231                        fsa_state2string(fsa_state),
 232                        fsa_cause2string(fsa_data->fsa_cause),
 233                        fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
 234         }
 235 
 236         /* start doing things... */
 237         s_crmd_fsa_actions(fsa_data);
 238         delete_fsa_input(fsa_data);
 239         fsa_data = NULL;
 240     }
 241 
 242     if (g_list_length(fsa_message_queue) > 0 || fsa_actions != A_NOTHING || do_fsa_stall) {
 243         crm_debug("Exiting the FSA: queue=%d, fsa_actions=0x%llx, stalled=%s",
 244                   g_list_length(fsa_message_queue), fsa_actions, do_fsa_stall ? "true" : "false");
 245     } else {
 246         crm_trace("Exiting the FSA");
 247     }
 248 
 249     /* cleanup inputs? */
 250     if (register_copy != fsa_input_register) {
 251         long long same = register_copy & fsa_input_register;
 252 
 253         fsa_dump_inputs(LOG_DEBUG, "Added", fsa_input_register ^ same);
 254         fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
 255     }
 256 
 257     fsa_dump_actions(fsa_actions, "Remaining");
 258     fsa_dump_queue(LOG_DEBUG);
 259 
 260     return fsa_state;
 261 }
 262 
 263 void
 264 s_crmd_fsa_actions(fsa_data_t * fsa_data)
     /*  */
 265 {
 266     /*
 267      * Process actions in order of priority but do only one
 268      * action at a time to avoid complicating the ordering.
 269      */
 270     CRM_CHECK(fsa_data != NULL, return);
 271     while (fsa_actions != A_NOTHING && do_fsa_stall == FALSE) {
 272 
 273         /* regular action processing in order of action priority
 274          *
 275          * Make sure all actions that connect to required systems
 276          * are performed first
 277          */
 278         if (fsa_actions & A_ERROR) {
 279             do_fsa_action(fsa_data, A_ERROR, do_log);
 280         } else if (fsa_actions & A_WARN) {
 281             do_fsa_action(fsa_data, A_WARN, do_log);
 282         } else if (fsa_actions & A_LOG) {
 283             do_fsa_action(fsa_data, A_LOG, do_log);
 284 
 285             /* get out of here NOW! before anything worse happens */
 286         } else if (fsa_actions & A_EXIT_1) {
 287             do_fsa_action(fsa_data, A_EXIT_1, do_exit);
 288 
 289             /* sub-system restart */
 290         } else if ((fsa_actions & O_LRM_RECONNECT) == O_LRM_RECONNECT) {
 291             do_fsa_action(fsa_data, O_LRM_RECONNECT, do_lrm_control);
 292         } else if ((fsa_actions & O_CIB_RESTART) == O_CIB_RESTART) {
 293             do_fsa_action(fsa_data, O_CIB_RESTART, do_cib_control);
 294         } else if ((fsa_actions & O_PE_RESTART) == O_PE_RESTART) {
 295             do_fsa_action(fsa_data, O_PE_RESTART, do_pe_control);
 296         } else if ((fsa_actions & O_TE_RESTART) == O_TE_RESTART) {
 297             do_fsa_action(fsa_data, O_TE_RESTART, do_te_control);
 298 
 299             /* essential start tasks */
 300         } else if (fsa_actions & A_STARTUP) {
 301             do_fsa_action(fsa_data, A_STARTUP, do_startup);
 302         } else if (fsa_actions & A_CIB_START) {
 303             do_fsa_action(fsa_data, A_CIB_START, do_cib_control);
 304         } else if (fsa_actions & A_HA_CONNECT) {
 305             do_fsa_action(fsa_data, A_HA_CONNECT, do_ha_control);
 306         } else if (fsa_actions & A_READCONFIG) {
 307             do_fsa_action(fsa_data, A_READCONFIG, do_read_config);
 308 
 309             /* sub-system start/connect */
 310         } else if (fsa_actions & A_LRM_CONNECT) {
 311             do_fsa_action(fsa_data, A_LRM_CONNECT, do_lrm_control);
 312         } else if (fsa_actions & A_CCM_CONNECT) {
 313 #if SUPPORT_HEARTBEAT
 314             if (is_heartbeat_cluster()) {
 315                 do_fsa_action(fsa_data, A_CCM_CONNECT, do_ccm_control);
 316             }
 317 #endif
 318             fsa_actions &= ~A_CCM_CONNECT;
 319 
 320         } else if (fsa_actions & A_TE_START) {
 321             do_fsa_action(fsa_data, A_TE_START, do_te_control);
 322         } else if (fsa_actions & A_PE_START) {
 323             do_fsa_action(fsa_data, A_PE_START, do_pe_control);
 324 
 325             /* Timers */
 326 /*              else if(fsa_actions & O_DC_TIMER_RESTART) {
 327                 do_fsa_action(fsa_data, O_DC_TIMER_RESTART,          do_timer_control) */ ;
 328         } else if (fsa_actions & A_DC_TIMER_STOP) {
 329             do_fsa_action(fsa_data, A_DC_TIMER_STOP, do_timer_control);
 330         } else if (fsa_actions & A_INTEGRATE_TIMER_STOP) {
 331             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_STOP, do_timer_control);
 332         } else if (fsa_actions & A_INTEGRATE_TIMER_START) {
 333             do_fsa_action(fsa_data, A_INTEGRATE_TIMER_START, do_timer_control);
 334         } else if (fsa_actions & A_FINALIZE_TIMER_STOP) {
 335             do_fsa_action(fsa_data, A_FINALIZE_TIMER_STOP, do_timer_control);
 336         } else if (fsa_actions & A_FINALIZE_TIMER_START) {
 337             do_fsa_action(fsa_data, A_FINALIZE_TIMER_START, do_timer_control);
 338 
 339             /*
 340              * Highest priority actions
 341              */
 342         } else if (fsa_actions & A_MSG_ROUTE) {
 343             do_fsa_action(fsa_data, A_MSG_ROUTE, do_msg_route);
 344         } else if (fsa_actions & A_RECOVER) {
 345             do_fsa_action(fsa_data, A_RECOVER, do_recover);
 346         } else if (fsa_actions & A_CL_JOIN_RESULT) {
 347             do_fsa_action(fsa_data, A_CL_JOIN_RESULT, do_cl_join_finalize_respond);
 348         } else if (fsa_actions & A_CL_JOIN_REQUEST) {
 349             do_fsa_action(fsa_data, A_CL_JOIN_REQUEST, do_cl_join_offer_respond);
 350         } else if (fsa_actions & A_SHUTDOWN_REQ) {
 351             do_fsa_action(fsa_data, A_SHUTDOWN_REQ, do_shutdown_req);
 352         } else if (fsa_actions & A_ELECTION_VOTE) {
 353             do_fsa_action(fsa_data, A_ELECTION_VOTE, do_election_vote);
 354         } else if (fsa_actions & A_ELECTION_COUNT) {
 355             do_fsa_action(fsa_data, A_ELECTION_COUNT, do_election_count_vote);
 356         } else if (fsa_actions & A_LRM_EVENT) {
 357             do_fsa_action(fsa_data, A_LRM_EVENT, do_lrm_event);
 358 
 359             /*
 360              * High priority actions
 361              */
 362         } else if (fsa_actions & A_STARTED) {
 363             do_fsa_action(fsa_data, A_STARTED, do_started);
 364         } else if (fsa_actions & A_CL_JOIN_QUERY) {
 365             do_fsa_action(fsa_data, A_CL_JOIN_QUERY, do_cl_join_query);
 366         } else if (fsa_actions & A_DC_TIMER_START) {
 367             do_fsa_action(fsa_data, A_DC_TIMER_START, do_timer_control);
 368 
 369             /*
 370              * Medium priority actions
 371              * - Membership
 372              */
 373         } else if (fsa_actions & A_DC_TAKEOVER) {
 374             do_fsa_action(fsa_data, A_DC_TAKEOVER, do_dc_takeover);
 375         } else if (fsa_actions & A_DC_RELEASE) {
 376             do_fsa_action(fsa_data, A_DC_RELEASE, do_dc_release);
 377         } else if (fsa_actions & A_DC_JOIN_FINAL) {
 378             do_fsa_action(fsa_data, A_DC_JOIN_FINAL, do_dc_join_final);
 379         } else if (fsa_actions & A_ELECTION_CHECK) {
 380             do_fsa_action(fsa_data, A_ELECTION_CHECK, do_election_check);
 381         } else if (fsa_actions & A_ELECTION_START) {
 382             do_fsa_action(fsa_data, A_ELECTION_START, do_election_vote);
 383         } else if (fsa_actions & A_DC_JOIN_OFFER_ALL) {
 384             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ALL, do_dc_join_offer_all);
 385         } else if (fsa_actions & A_DC_JOIN_OFFER_ONE) {
 386             do_fsa_action(fsa_data, A_DC_JOIN_OFFER_ONE, do_dc_join_offer_one);
 387         } else if (fsa_actions & A_DC_JOIN_PROCESS_REQ) {
 388             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_REQ, do_dc_join_filter_offer);
 389         } else if (fsa_actions & A_DC_JOIN_PROCESS_ACK) {
 390             do_fsa_action(fsa_data, A_DC_JOIN_PROCESS_ACK, do_dc_join_ack);
 391         } else if (fsa_actions & A_DC_JOIN_FINALIZE) {
 392             do_fsa_action(fsa_data, A_DC_JOIN_FINALIZE, do_dc_join_finalize);
 393         } else if (fsa_actions & A_CL_JOIN_ANNOUNCE) {
 394             do_fsa_action(fsa_data, A_CL_JOIN_ANNOUNCE, do_cl_join_announce);
 395 
 396             /*
 397              * Low(er) priority actions
 398              * Make sure the CIB is always updated before invoking the
 399              * PE, and the PE before the TE
 400              */
 401         } else if (fsa_actions & A_TE_HALT) {
 402             do_fsa_action(fsa_data, A_TE_HALT, do_te_invoke);
 403         } else if (fsa_actions & A_TE_CANCEL) {
 404             do_fsa_action(fsa_data, A_TE_CANCEL, do_te_invoke);
 405         } else if (fsa_actions & A_LRM_INVOKE) {
 406             do_fsa_action(fsa_data, A_LRM_INVOKE, do_lrm_invoke);
 407         } else if (fsa_actions & A_PE_INVOKE) {
 408             do_fsa_action(fsa_data, A_PE_INVOKE, do_pe_invoke);
 409         } else if (fsa_actions & A_TE_INVOKE) {
 410             do_fsa_action(fsa_data, A_TE_INVOKE, do_te_invoke);
 411 
 412             /* Shutdown actions */
 413         } else if (fsa_actions & A_DC_RELEASED) {
 414             do_fsa_action(fsa_data, A_DC_RELEASED, do_dc_release);
 415         } else if (fsa_actions & A_PE_STOP) {
 416             do_fsa_action(fsa_data, A_PE_STOP, do_pe_control);
 417         } else if (fsa_actions & A_TE_STOP) {
 418             do_fsa_action(fsa_data, A_TE_STOP, do_te_control);
 419         } else if (fsa_actions & A_SHUTDOWN) {
 420             do_fsa_action(fsa_data, A_SHUTDOWN, do_shutdown);
 421         } else if (fsa_actions & A_LRM_DISCONNECT) {
 422             do_fsa_action(fsa_data, A_LRM_DISCONNECT, do_lrm_control);
 423         } else if (fsa_actions & A_CCM_DISCONNECT) {
 424 #if SUPPORT_HEARTBEAT
 425             if (is_heartbeat_cluster()) {
 426                 do_fsa_action(fsa_data, A_CCM_DISCONNECT, do_ccm_control);
 427             }
 428 #endif
 429             fsa_actions &= ~A_CCM_DISCONNECT;
 430 
 431         } else if (fsa_actions & A_HA_DISCONNECT) {
 432             do_fsa_action(fsa_data, A_HA_DISCONNECT, do_ha_control);
 433         } else if (fsa_actions & A_CIB_STOP) {
 434             do_fsa_action(fsa_data, A_CIB_STOP, do_cib_control);
 435         } else if (fsa_actions & A_STOP) {
 436             do_fsa_action(fsa_data, A_STOP, do_stop);
 437 
 438             /* exit gracefully */
 439         } else if (fsa_actions & A_EXIT_0) {
 440             do_fsa_action(fsa_data, A_EXIT_0, do_exit);
 441 
 442             /* Error checking and reporting */
 443         } else {
 444             crm_err("Action %s not supported "CRM_XS" 0x%llx",
 445                     fsa_action2string(fsa_actions), fsa_actions);
 446             register_fsa_error_adv(C_FSA_INTERNAL, I_ERROR, fsa_data, NULL, __FUNCTION__);
 447         }
 448     }
 449 }
 450 
 451 void
 452 log_fsa_input(fsa_data_t * stored_msg)
     /*  */
 453 {
 454     CRM_ASSERT(stored_msg);
 455     crm_trace("Processing queued input %d", stored_msg->id);
 456     if (stored_msg->fsa_cause == C_CCM_CALLBACK) {
 457         crm_trace("FSA processing CCM callback from %s", stored_msg->origin);
 458 
 459     } else if (stored_msg->fsa_cause == C_LRM_OP_CALLBACK) {
 460         crm_trace("FSA processing LRM callback from %s", stored_msg->origin);
 461 
 462     } else if (stored_msg->data == NULL) {
 463         crm_trace("FSA processing input from %s", stored_msg->origin);
 464 
 465     } else {
 466         ha_msg_input_t *ha_input = fsa_typed_data_adv(stored_msg, fsa_dt_ha_msg, __FUNCTION__);
 467 
 468         crm_trace("FSA processing XML message from %s", stored_msg->origin);
 469         crm_log_xml_trace(ha_input->xml, "FSA message data");
 470     }
 471 }
 472 
 473 long long
 474 do_state_transition(long long actions,
     /*  */
 475                     enum crmd_fsa_state cur_state,
 476                     enum crmd_fsa_state next_state, fsa_data_t * msg_data)
 477 {
 478     int level = LOG_INFO;
 479     long long tmp = actions;
 480     gboolean clear_recovery_bit = TRUE;
 481 
 482     enum crmd_fsa_cause cause = msg_data->fsa_cause;
 483     enum crmd_fsa_input current_input = msg_data->fsa_input;
 484 
 485     const char *state_from = fsa_state2string(cur_state);
 486     const char *state_to = fsa_state2string(next_state);
 487     const char *input = fsa_input2string(current_input);
 488 
 489     CRM_LOG_ASSERT(cur_state != next_state);
 490 
 491     do_dot_log(DOT_PREFIX "\t%s -> %s [ label=%s cause=%s origin=%s ]",
 492                state_from, state_to, input, fsa_cause2string(cause), msg_data->origin);
 493 
 494     if (cur_state == S_IDLE || next_state == S_IDLE) {
 495         level = LOG_NOTICE;
 496     } else if (cur_state == S_NOT_DC || next_state == S_NOT_DC) {
 497         level = LOG_NOTICE;
 498     } else if (cur_state == S_ELECTION) {
 499         level = LOG_NOTICE;
 500     } else if (cur_state == S_STARTING) {
 501         level = LOG_NOTICE;
 502     } else if (next_state == S_RECOVERY) {
 503         level = LOG_WARNING;
 504     }
 505 
 506     do_crm_log(level, "State transition %s -> %s "
 507                CRM_XS " input=%s cause=%s origin=%s",
 508                state_from, state_to, input, fsa_cause2string(cause),
 509                msg_data->origin);
 510 
 511     /* the last two clauses might cause trouble later */
 512     if (next_state != S_ELECTION && cur_state != S_RELEASE_DC) {
 513         election_timeout_stop(fsa_election);
 514 /*      } else { */
 515 /*              crm_timer_start(election_timeout); */
 516     }
 517 #if 0
 518     if ((fsa_input_register & R_SHUTDOWN)) {
 519         set_bit(tmp, A_DC_TIMER_STOP);
 520     }
 521 #endif
 522     if (next_state == S_INTEGRATION) {
 523         set_bit(tmp, A_INTEGRATE_TIMER_START);
 524     } else {
 525         set_bit(tmp, A_INTEGRATE_TIMER_STOP);
 526     }
 527 
 528     if (next_state == S_FINALIZE_JOIN) {
 529         set_bit(tmp, A_FINALIZE_TIMER_START);
 530     } else {
 531         set_bit(tmp, A_FINALIZE_TIMER_STOP);
 532     }
 533 
 534     if (next_state != S_PENDING) {
 535         set_bit(tmp, A_DC_TIMER_STOP);
 536     }
 537     if (next_state != S_ELECTION) {
 538         highest_born_on = 0;
 539     }
 540     if (next_state != S_IDLE) {
 541         crm_timer_stop(recheck_timer);
 542     }
 543 
 544     if (cur_state == S_FINALIZE_JOIN && next_state == S_POLICY_ENGINE) {
 545         populate_cib_nodes(node_update_quick|node_update_all, __FUNCTION__);
 546     }
 547 
 548     switch (next_state) {
 549         case S_PENDING:
 550             fsa_cib_conn->cmds->set_slave(fsa_cib_conn, cib_scope_local);
 551             /* fall through */
 552         case S_ELECTION:
 553             crm_trace("Resetting our DC to NULL on transition to %s", fsa_state2string(next_state));
 554             update_dc(NULL);
 555             break;
 556         case S_NOT_DC:
 557             election_trigger->counter = 0;
 558             purge_stonith_cleanup();
 559 
 560             if (is_set(fsa_input_register, R_SHUTDOWN)) {
 561                 crm_info("(Re)Issuing shutdown request now" " that we have a new DC");
 562                 set_bit(tmp, A_SHUTDOWN_REQ);
 563             }
 564             CRM_LOG_ASSERT(fsa_our_dc != NULL);
 565             if (fsa_our_dc == NULL) {
 566                 crm_err("Reached S_NOT_DC without a DC" " being recorded");
 567             }
 568             break;
 569         case S_RECOVERY:
 570             clear_recovery_bit = FALSE;
 571             break;
 572 
 573         case S_FINALIZE_JOIN:
 574             CRM_LOG_ASSERT(AM_I_DC);
 575             if (cause == C_TIMER_POPPED) {
 576                 crm_warn("Progressed to state %s after %s",
 577                          fsa_state2string(next_state), fsa_cause2string(cause));
 578             }
 579             if (crmd_join_phase_count(crm_join_welcomed) > 0) {
 580                 crm_warn("%u cluster nodes failed to respond"
 581                          " to the join offer.", crmd_join_phase_count(crm_join_welcomed));
 582                 crmd_join_phase_log(LOG_NOTICE);
 583 
 584             } else {
 585                 crm_debug("All %d cluster nodes responded to the join offer.",
 586                           crmd_join_phase_count(crm_join_integrated));
 587             }
 588             break;
 589 
 590         case S_POLICY_ENGINE:
 591             election_trigger->counter = 0;
 592             CRM_LOG_ASSERT(AM_I_DC);
 593             if (cause == C_TIMER_POPPED) {
 594                 crm_info("Progressed to state %s after %s",
 595                          fsa_state2string(next_state), fsa_cause2string(cause));
 596             }
 597 
 598             if (crmd_join_phase_count(crm_join_finalized) > 0) {
 599                 crm_err("%u cluster nodes failed to confirm their join.",
 600                         crmd_join_phase_count(crm_join_finalized));
 601                 crmd_join_phase_log(LOG_NOTICE);
 602 
 603             } else if (crmd_join_phase_count(crm_join_confirmed)
 604                        == crm_active_peers()) {
 605                 crm_debug("All %u cluster nodes are"
 606                           " eligible to run resources.", crm_active_peers());
 607 
 608             } else if (crmd_join_phase_count(crm_join_confirmed) > crm_active_peers()) {
 609                 crm_err("We have more confirmed nodes than our membership does: %d vs. %d",
 610                         crmd_join_phase_count(crm_join_confirmed), crm_active_peers());
 611                 register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
 612 
 613             } else if (saved_ccm_membership_id != crm_peer_seq) {
 614                 crm_info("Membership changed: %llu -> %llu - join restart",
 615                          saved_ccm_membership_id, crm_peer_seq);
 616                 register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
 617 
 618             } else {
 619                 crm_warn("Only %u of %u cluster "
 620                          "nodes are eligible to run resources - continue %d",
 621                          crmd_join_phase_count(crm_join_confirmed),
 622                          crm_active_peers(), crmd_join_phase_count(crm_join_welcomed));
 623             }
 624 /*                      initialize_join(FALSE); */
 625             break;
 626 
 627         case S_STOPPING:
 628         case S_TERMINATE:
 629             /* possibly redundant */
 630             set_bit(fsa_input_register, R_SHUTDOWN);
 631             break;
 632 
 633         case S_IDLE:
 634             CRM_LOG_ASSERT(AM_I_DC);
 635             dump_rsc_info();
 636             if (is_set(fsa_input_register, R_SHUTDOWN)) {
 637                 crm_info("(Re)Issuing shutdown request now" " that we are the DC");
 638                 set_bit(tmp, A_SHUTDOWN_REQ);
 639             }
 640             if (recheck_timer->period_ms > 0) {
 641                 crm_debug("Starting %s", get_timer_desc(recheck_timer));
 642                 crm_timer_start(recheck_timer);
 643             }
 644             break;
 645 
 646         default:
 647             break;
 648     }
 649 
 650     if (clear_recovery_bit && next_state != S_PENDING) {
 651         tmp &= ~A_RECOVER;
 652     } else if (clear_recovery_bit == FALSE) {
 653         tmp |= A_RECOVER;
 654     }
 655 
 656     if (tmp != actions) {
 657         /* fsa_dump_actions(actions ^ tmp, "New actions"); */
 658         actions = tmp;
 659     }
 660 
 661     return actions;
 662 }
 663 
 664 void
 665 dump_rsc_info(void)
     /*  */
 666 {
 667 }
 668 
 669 void
 670 ghash_print_node(gpointer key, gpointer value, gpointer user_data)
     /*  */
 671 {
 672     const char *text = user_data;
 673     const char *uname = key;
 674     const char *value_s = value;
 675 
 676     crm_info("%s: %s %s", text, uname, value_s);
 677 }
/* */
root/crmd/fsa.c

DEFINITIONS
