daemons/controld/controld

/* */
This source file includes following definitions.
get_timer_desc
controld_stop_timer
controld_start_timer
do_timer_control
crm_timer_popped
controld_init_fsa_timers
controld_configure_fsa_timers
controld_free_fsa_timers
controld_is_started_transition_timer
controld_start_recheck_timer
controld_start_wait_timer
controld_stop_recheck_timer
controld_get_period_transition_timer
controld_reset_counter_election_timer
controld_stop_transition_timer
controld_start_transition_timer
controld_shutdown_start_countdown
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <time.h>
  13 #include <stdlib.h>
  14 
  15 #include <crm/crm.h>
  16 #include <crm/common/xml.h>
  17 #include <pacemaker-controld.h>
  18 
  19 //! FSA mainloop timer type
  20 typedef struct fsa_timer_s {
  21     guint source_id;                        //!< Timer source ID
  22     guint period_ms;                        //!< Timer period
  23     enum crmd_fsa_input fsa_input;          //!< Input to register if timer pops
  24     gboolean (*callback) (gpointer data);   //!< What do if timer pops
  25     bool log_error;                         //!< Timer popping indicates error
  26     int counter;                            //!< For detecting loops
  27 } fsa_timer_t;
  28 
  29 //! Wait before retrying a failed cib or executor connection
  30 static fsa_timer_t *wait_timer = NULL;
  31 
  32 //! Periodically re-run scheduler (for date_spec evaluation and as a failsafe)
  33 static fsa_timer_t *recheck_timer = NULL;
  34 
  35 //! Wait at start-up, or after an election, for DC to make contact
  36 static fsa_timer_t *election_timer = NULL;
  37 
  38 //! Delay start of new transition with expectation something else might happen
  39 static fsa_timer_t *transition_timer = NULL;
  40 
  41 //! \c PCMK_OPT_JOIN_INTEGRATION_TIMEOUT
  42 static fsa_timer_t *integration_timer = NULL;
  43 
  44 //! \c PCMK_OPT_JOIN_FINALIZATION_TIMEOUT
  45 static fsa_timer_t *finalization_timer = NULL;
  46 
  47 // Wait for DC to stop all resources and give us the all-clear to shut down
  48 fsa_timer_t *shutdown_escalation_timer = NULL;
  49 
  50 //! Cluster recheck interval (from configuration)
  51 static guint recheck_interval_ms = 0;
  52 
  53 static const char *
  54 get_timer_desc(fsa_timer_t * timer)
     /*  */
  55 {
  56     if (timer == election_timer) {
  57         return "Election Trigger";
  58 
  59     } else if (timer == shutdown_escalation_timer) {
  60         return "Shutdown Escalation";
  61 
  62     } else if (timer == integration_timer) {
  63         return "Integration Timer";
  64 
  65     } else if (timer == finalization_timer) {
  66         return "Finalization Timer";
  67 
  68     } else if (timer == transition_timer) {
  69         return "New Transition Timer";
  70 
  71     } else if (timer == wait_timer) {
  72         return "Wait Timer";
  73 
  74     } else if (timer == recheck_timer) {
  75         return "Cluster Recheck Timer";
  76 
  77     }
  78     return "Unknown Timer";
  79 }
  80 
  81 /*!
  82  * \internal
  83  * \brief Stop an FSA timer
  84  *
  85  * \param[in,out] timer  Timer to stop
  86  *
  87  * \return true if the timer was running, or false otherwise
  88  */
  89 static bool
  90 controld_stop_timer(fsa_timer_t *timer)
     /*  */
  91 {
  92     CRM_CHECK(timer != NULL, return false);
  93 
  94     if (timer->source_id != 0) {
  95         crm_trace("Stopping %s (would inject %s if popped after %ums, src=%d)",
  96                   get_timer_desc(timer), fsa_input2string(timer->fsa_input),
  97                   timer->period_ms, timer->source_id);
  98         g_source_remove(timer->source_id);
  99         timer->source_id = 0;
 100 
 101     } else {
 102         crm_trace("%s already stopped (would inject %s if popped after %ums)",
 103                   get_timer_desc(timer), fsa_input2string(timer->fsa_input),
 104                   timer->period_ms);
 105         return false;
 106     }
 107     return true;
 108 }
 109 
 110 /*!
 111  * \internal
 112  * \brief Start an FSA timer
 113  *
 114  * \param[in,out] timer  Timer to start
 115  */
 116 static void
 117 controld_start_timer(fsa_timer_t *timer)
     /*  */
 118 {
 119     if (timer->source_id == 0 && timer->period_ms > 0) {
 120         timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
 121         CRM_ASSERT(timer->source_id != 0);
 122         crm_debug("Started %s (inject %s if pops after %ums, source=%d)",
 123                   get_timer_desc(timer), fsa_input2string(timer->fsa_input),
 124                   timer->period_ms, timer->source_id);
 125     } else {
 126         crm_debug("%s already running (inject %s if pops after %ums, source=%d)",
 127                   get_timer_desc(timer), fsa_input2string(timer->fsa_input),
 128                   timer->period_ms, timer->source_id);
 129     }
 130 }
 131 
 132 /*      A_DC_TIMER_STOP, A_DC_TIMER_START,
 133  *      A_FINALIZE_TIMER_STOP, A_FINALIZE_TIMER_START
 134  *      A_INTEGRATE_TIMER_STOP, A_INTEGRATE_TIMER_START
 135  */
 136 void
 137 do_timer_control(long long action,
     /*  */
 138                  enum crmd_fsa_cause cause,
 139                  enum crmd_fsa_state cur_state,
 140                  enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 141 {
 142     gboolean timer_op_ok = TRUE;
 143 
 144     if (action & A_DC_TIMER_STOP) {
 145         timer_op_ok = controld_stop_timer(election_timer);
 146 
 147     } else if (action & A_FINALIZE_TIMER_STOP) {
 148         timer_op_ok = controld_stop_timer(finalization_timer);
 149 
 150     } else if (action & A_INTEGRATE_TIMER_STOP) {
 151         timer_op_ok = controld_stop_timer(integration_timer);
 152     }
 153 
 154     /* don't start a timer that wasn't already running */
 155     if (action & A_DC_TIMER_START && timer_op_ok) {
 156         controld_start_timer(election_timer);
 157         if (AM_I_DC) {
 158             /* there can be only one */
 159             register_fsa_input(cause, I_ELECTION, NULL);
 160         }
 161 
 162     } else if (action & A_FINALIZE_TIMER_START) {
 163         controld_start_timer(finalization_timer);
 164 
 165     } else if (action & A_INTEGRATE_TIMER_START) {
 166         controld_start_timer(integration_timer);
 167     }
 168 }
 169 
 170 static gboolean
 171 crm_timer_popped(gpointer data)
     /*  */
 172 {
 173     fsa_timer_t *timer = (fsa_timer_t *) data;
 174 
 175     if (timer->log_error) {
 176         crm_err("%s just popped in state %s! " CRM_XS " input=%s time=%ums",
 177                 get_timer_desc(timer),
 178                 fsa_state2string(controld_globals.fsa_state),
 179                 fsa_input2string(timer->fsa_input), timer->period_ms);
 180     } else {
 181         crm_info("%s just popped " CRM_XS " input=%s time=%ums",
 182                  get_timer_desc(timer), fsa_input2string(timer->fsa_input),
 183                  timer->period_ms);
 184         timer->counter++;
 185     }
 186 
 187     if ((timer == election_timer) && (election_timer->counter > 5)) {
 188         crm_notice("We appear to be in an election loop, something may be wrong");
 189         crm_write_blackbox(0, NULL);
 190         election_timer->counter = 0;
 191     }
 192 
 193     controld_stop_timer(timer);  // Make timer _not_ go off again
 194 
 195     if (timer->fsa_input == I_INTEGRATED) {
 196         crm_info("Welcomed: %d, Integrated: %d",
 197                  crmd_join_phase_count(crm_join_welcomed),
 198                  crmd_join_phase_count(crm_join_integrated));
 199         if (crmd_join_phase_count(crm_join_welcomed) == 0) {
 200             // If we don't even have ourselves, start again
 201             register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL,
 202                                    __func__);
 203 
 204         } else {
 205             register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
 206         }
 207 
 208     } else if ((timer == recheck_timer)
 209                && (controld_globals.fsa_state != S_IDLE)) {
 210         crm_debug("Discarding %s event in state: %s",
 211                   fsa_input2string(timer->fsa_input),
 212                   fsa_state2string(controld_globals.fsa_state));
 213 
 214     } else if ((timer == finalization_timer)
 215                && (controld_globals.fsa_state != S_FINALIZE_JOIN)) {
 216         crm_debug("Discarding %s event in state: %s",
 217                   fsa_input2string(timer->fsa_input),
 218                   fsa_state2string(controld_globals.fsa_state));
 219 
 220     } else if (timer->fsa_input != I_NULL) {
 221         register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
 222     }
 223 
 224     controld_trigger_fsa();
 225 
 226     return TRUE;
 227 }
 228 
 229 bool
 230 controld_init_fsa_timers(void)
     /*  */
 231 {
 232     transition_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 233     integration_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 234     finalization_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 235     election_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 236     shutdown_escalation_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 237     wait_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 238     recheck_timer = pcmk__assert_alloc(1, sizeof(fsa_timer_t));
 239 
 240     election_timer->source_id = 0;
 241     election_timer->period_ms = 0;
 242     election_timer->fsa_input = I_DC_TIMEOUT;
 243     election_timer->callback = crm_timer_popped;
 244     election_timer->log_error = FALSE;
 245 
 246     transition_timer->source_id = 0;
 247     transition_timer->period_ms = 0;
 248     transition_timer->fsa_input = I_PE_CALC;
 249     transition_timer->callback = crm_timer_popped;
 250     transition_timer->log_error = FALSE;
 251 
 252     integration_timer->source_id = 0;
 253     integration_timer->period_ms = 0;
 254     integration_timer->fsa_input = I_INTEGRATED;
 255     integration_timer->callback = crm_timer_popped;
 256     integration_timer->log_error = TRUE;
 257 
 258     finalization_timer->source_id = 0;
 259     finalization_timer->period_ms = 0;
 260     finalization_timer->fsa_input = I_FINALIZED;
 261     finalization_timer->callback = crm_timer_popped;
 262     finalization_timer->log_error = FALSE;
 263 
 264     /* We can't use I_FINALIZED here, because that creates a bug in the join
 265      * process where a joining node can be stuck in S_PENDING while we think it
 266      * is in S_NOT_DC. This created an infinite transition loop in which we
 267      * continually send probes which the node NACKs because it's pending.
 268      *
 269      * If we have nodes where the cluster layer is active but the controller is
 270      * not, we can avoid this causing an election/join loop, in the integration
 271      * phase.
 272      */
 273     finalization_timer->fsa_input = I_ELECTION;
 274 
 275     shutdown_escalation_timer->source_id = 0;
 276     shutdown_escalation_timer->period_ms = 0;
 277     shutdown_escalation_timer->fsa_input = I_STOP;
 278     shutdown_escalation_timer->callback = crm_timer_popped;
 279     shutdown_escalation_timer->log_error = TRUE;
 280 
 281     wait_timer->source_id = 0;
 282     wait_timer->period_ms = 2000;
 283     wait_timer->fsa_input = I_NULL;
 284     wait_timer->callback = crm_timer_popped;
 285     wait_timer->log_error = FALSE;
 286 
 287     recheck_timer->source_id = 0;
 288     recheck_timer->period_ms = 0;
 289     recheck_timer->fsa_input = I_PE_CALC;
 290     recheck_timer->callback = crm_timer_popped;
 291     recheck_timer->log_error = FALSE;
 292 
 293     return TRUE;
 294 }
 295 
 296 /*!
 297  * \internal
 298  * \brief Configure timers based on the CIB
 299  *
 300  * \param[in,out] options  Name/value pairs for configured options
 301  */
 302 void
 303 controld_configure_fsa_timers(GHashTable *options)
     /*  */
 304 {
 305     const char *value = NULL;
 306 
 307     // Election timer
 308     value = g_hash_table_lookup(options, PCMK_OPT_DC_DEADTIME);
 309     pcmk_parse_interval_spec(value, &(election_timer->period_ms));
 310 
 311     // Integration timer
 312     value = g_hash_table_lookup(options, PCMK_OPT_JOIN_INTEGRATION_TIMEOUT);
 313     pcmk_parse_interval_spec(value, &(integration_timer->period_ms));
 314 
 315     // Finalization timer
 316     value = g_hash_table_lookup(options, PCMK_OPT_JOIN_FINALIZATION_TIMEOUT);
 317     pcmk_parse_interval_spec(value, &(finalization_timer->period_ms));
 318 
 319     // Shutdown escalation timer
 320     value = g_hash_table_lookup(options, PCMK_OPT_SHUTDOWN_ESCALATION);
 321     pcmk_parse_interval_spec(value, &(shutdown_escalation_timer->period_ms));
 322     crm_debug("Shutdown escalation occurs if DC has not responded to request "
 323               "in %ums", shutdown_escalation_timer->period_ms);
 324 
 325     // Transition timer
 326     value = g_hash_table_lookup(options, PCMK_OPT_TRANSITION_DELAY);
 327     pcmk_parse_interval_spec(value, &(transition_timer->period_ms));
 328 
 329     // Recheck interval
 330     value = g_hash_table_lookup(options, PCMK_OPT_CLUSTER_RECHECK_INTERVAL);
 331     pcmk_parse_interval_spec(value, &recheck_interval_ms);
 332     crm_debug("Re-run scheduler after %dms of inactivity", recheck_interval_ms);
 333 }
 334 
 335 void
 336 controld_free_fsa_timers(void)
     /*  */
 337 {
 338     controld_stop_timer(transition_timer);
 339     controld_stop_timer(integration_timer);
 340     controld_stop_timer(finalization_timer);
 341     controld_stop_timer(election_timer);
 342     controld_stop_timer(shutdown_escalation_timer);
 343     controld_stop_timer(wait_timer);
 344     controld_stop_timer(recheck_timer);
 345 
 346     free(transition_timer); transition_timer = NULL;
 347     free(integration_timer); integration_timer = NULL;
 348     free(finalization_timer); finalization_timer = NULL;
 349     free(election_timer); election_timer = NULL;
 350     free(shutdown_escalation_timer); shutdown_escalation_timer = NULL;
 351     free(wait_timer); wait_timer = NULL;
 352     free(recheck_timer); recheck_timer = NULL;
 353 }
 354 
 355 /*!
 356  * \internal
 357  * \brief Check whether the transition timer is started
 358  * \return true if the transition timer is started, or false otherwise
 359  */
 360 bool
 361 controld_is_started_transition_timer(void)
     /*  */
 362 {
 363     return (transition_timer->period_ms > 0)
 364            && (transition_timer->source_id != 0);
 365 }
 366 
 367 /*!
 368  * \internal
 369  * \brief Start the recheck timer
 370  */
 371 void
 372 controld_start_recheck_timer(void)
     /*  */
 373 {
 374     // Default to recheck interval configured in CIB (if any)
 375     guint period_ms = recheck_interval_ms;
 376 
 377     // If scheduler supplied a "recheck by" time, check whether that's sooner
 378     if (controld_globals.transition_graph->recheck_by > 0) {
 379         time_t diff_seconds = controld_globals.transition_graph->recheck_by
 380                               - time(NULL);
 381 
 382         if (diff_seconds < 1) {
 383             // We're already past the desired time
 384             period_ms = 500;
 385         } else {
 386             period_ms = (guint) diff_seconds * 1000;
 387         }
 388 
 389         // Use "recheck by" only if it's sooner than interval from CIB
 390         if (period_ms > recheck_interval_ms) {
 391             period_ms = recheck_interval_ms;
 392         }
 393     }
 394 
 395     if (period_ms > 0) {
 396         recheck_timer->period_ms = period_ms;
 397         controld_start_timer(recheck_timer);
 398     }
 399 }
 400 
 401 /*!
 402  * \internal
 403  * \brief Start the wait timer
 404  */
 405 void
 406 controld_start_wait_timer(void)
     /*  */
 407 {
 408     controld_start_timer(wait_timer);
 409 }
 410 
 411 /*!
 412  * \internal
 413  * \brief Stop the recheck timer
 414  *
 415  * \return true if the recheck timer was running, or false otherwise
 416  */
 417 bool
 418 controld_stop_recheck_timer(void)
     /*  */
 419 {
 420     return controld_stop_timer(recheck_timer);
 421 }
 422 
 423 /*!
 424  * \brief Get the transition timer's configured period
 425  * \return The transition_timer's period
 426  */
 427 guint
 428 controld_get_period_transition_timer(void)
     /*  */
 429 {
 430     return transition_timer->period_ms;
 431 }
 432 
 433 /*!
 434  * \internal
 435  * \brief Reset the election timer's counter to 0
 436  */
 437 void
 438 controld_reset_counter_election_timer(void)
     /*  */
 439 {
 440     election_timer->counter = 0;
 441 }
 442 
 443 /*!
 444  * \internal
 445  * \brief Stop the transition timer
 446  *
 447  * \return true if the transition timer was running, or false otherwise
 448  */
 449 bool
 450 controld_stop_transition_timer(void)
     /*  */
 451 {
 452     return controld_stop_timer(transition_timer);
 453 }
 454 
 455 /*!
 456  * \internal
 457  * \brief Start the transition timer
 458  */
 459 void
 460 controld_start_transition_timer(void)
     /*  */
 461 {
 462     controld_start_timer(transition_timer);
 463 }
 464 
 465 /*!
 466  * \internal
 467  * \brief Start the countdown sequence for a shutdown
 468  *
 469  * \param[in] default_period_ms  Period to use if the shutdown escalation
 470  *                               timer's period is 0
 471  */
 472 void
 473 controld_shutdown_start_countdown(guint default_period_ms)
     /*  */
 474 {
 475     if (shutdown_escalation_timer->period_ms == 0) {
 476         shutdown_escalation_timer->period_ms = default_period_ms;
 477     }
 478 
 479     crm_notice("Initiating controller shutdown sequence " CRM_XS " limit=%ums",
 480                shutdown_escalation_timer->period_ms);
 481     controld_start_timer(shutdown_escalation_timer);
 482 }
/* */
root/daemons/controld/controld_timers.c

DEFINITIONS
