lib/pengine/pe_actions.c

/* */
This source file includes following definitions.
add_singleton
lookup_singleton
find_existing_action
find_exact_action_config
pcmk__find_action_config
new_action
pcmk__unpack_action_rsc_params
update_action_optional
effective_quorum_policy
update_resource_action_runnable
valid_stop_on_fail
validate_on_fail
unpack_timeout
unpack_interval_origin
unpack_start_delay
most_frequent_monitor
pcmk__unpack_action_meta
pcmk__action_requires
pcmk__parse_on_fail
pcmk__role_after_failure
unpack_operation
custom_action
get_pseudo_op
find_unfencing_devices
node_priority_fencing_delay
pe_fence_op
pe_free_action
get_complex_task
find_first_action
find_actions
find_actions_exact
pe__resource_actions
pe__action2reason
pe_action_set_reason
pe__clear_resource_history
pe__is_newer_op
sort_op_by_callid
pe__new_rsc_pseudo_action
pe__add_action_expected_result
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <glib.h>
  13 #include <stdbool.h>
  14 
  15 #include <crm/crm.h>
  16 #include <crm/common/xml.h>
  17 #include <crm/common/scheduler_internal.h>
  18 #include <crm/pengine/internal.h>
  19 #include <crm/common/xml_internal.h>
  20 #include "pe_status_private.h"
  21 
  22 static void unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
  23                              guint interval_ms);
  24 
  25 static void
  26 add_singleton(pcmk_scheduler_t *scheduler, pcmk_action_t *action)
     /*  */
  27 {
  28     if (scheduler->priv->singletons == NULL) {
  29         scheduler->priv->singletons = pcmk__strkey_table(NULL, NULL);
  30     }
  31     g_hash_table_insert(scheduler->priv->singletons, action->uuid, action);
  32 }
  33 
  34 static pcmk_action_t *
  35 lookup_singleton(pcmk_scheduler_t *scheduler, const char *action_uuid)
     /*  */
  36 {
  37     /* @TODO This is the only use of the pcmk_scheduler_t:singletons hash table.
  38      * Compare the performance of this approach to keeping the
  39      * pcmk_scheduler_t:actions list sorted by action key and just searching
  40      * that instead.
  41      */
  42     if (scheduler->priv->singletons == NULL) {
  43         return NULL;
  44     }
  45     return g_hash_table_lookup(scheduler->priv->singletons, action_uuid);
  46 }
  47 
  48 /*!
  49  * \internal
  50  * \brief Find an existing action that matches arguments
  51  *
  52  * \param[in] key        Action key to match
  53  * \param[in] rsc        Resource to match (if any)
  54  * \param[in] node       Node to match (if any)
  55  * \param[in] scheduler  Scheduler data
  56  *
  57  * \return Existing action that matches arguments (or NULL if none)
  58  */
  59 static pcmk_action_t *
  60 find_existing_action(const char *key, const pcmk_resource_t *rsc,
     /*  */
  61                      const pcmk_node_t *node, const pcmk_scheduler_t *scheduler)
  62 {
  63     /* When rsc is NULL, it would be quicker to check
  64      * scheduler->priv->singletons, but checking all scheduler->priv->actions
  65      * takes the node into account.
  66      */
  67     GList *actions = (rsc == NULL)? scheduler->priv->actions : rsc->priv->actions;
  68     GList *matches = find_actions(actions, key, node);
  69     pcmk_action_t *action = NULL;
  70 
  71     if (matches == NULL) {
  72         return NULL;
  73     }
  74     CRM_LOG_ASSERT(!pcmk__list_of_multiple(matches));
  75 
  76     action = matches->data;
  77     g_list_free(matches);
  78     return action;
  79 }
  80 
  81 /*!
  82  * \internal
  83  * \brief Find the XML configuration corresponding to a specific action key
  84  *
  85  * \param[in] rsc               Resource to find action configuration for
  86  * \param[in] key               "RSC_ACTION_INTERVAL" of action to find
  87  * \param[in] include_disabled  If false, do not return disabled actions
  88  *
  89  * \return XML configuration of desired action if any, otherwise NULL
  90  */
  91 static xmlNode *
  92 find_exact_action_config(const pcmk_resource_t *rsc, const char *action_name,
     /*  */
  93                          guint interval_ms, bool include_disabled)
  94 {
  95     for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
  96                                                    PCMK_XE_OP, NULL, NULL);
  97          operation != NULL; operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
  98 
  99         bool enabled = false;
 100         const char *config_name = NULL;
 101         const char *interval_spec = NULL;
 102         guint tmp_ms = 0U;
 103 
 104         // @TODO This does not consider meta-attributes, rules, defaults, etc.
 105         if (!include_disabled
 106             && (pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
 107                                        &enabled) == pcmk_rc_ok) && !enabled) {
 108             continue;
 109         }
 110 
 111         interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
 112         pcmk_parse_interval_spec(interval_spec, &tmp_ms);
 113         if (tmp_ms != interval_ms) {
 114             continue;
 115         }
 116 
 117         config_name = crm_element_value(operation, PCMK_XA_NAME);
 118         if (pcmk__str_eq(action_name, config_name, pcmk__str_none)) {
 119             return operation;
 120         }
 121     }
 122     return NULL;
 123 }
 124 
 125 /*!
 126  * \internal
 127  * \brief Find the XML configuration of a resource action
 128  *
 129  * \param[in] rsc               Resource to find action configuration for
 130  * \param[in] action_name       Action name to search for
 131  * \param[in] interval_ms       Action interval (in milliseconds) to search for
 132  * \param[in] include_disabled  If false, do not return disabled actions
 133  *
 134  * \return XML configuration of desired action if any, otherwise NULL
 135  */
 136 xmlNode *
 137 pcmk__find_action_config(const pcmk_resource_t *rsc, const char *action_name,
     /*  */
 138                          guint interval_ms, bool include_disabled)
 139 {
 140     xmlNode *action_config = NULL;
 141 
 142     // Try requested action first
 143     action_config = find_exact_action_config(rsc, action_name, interval_ms,
 144                                              include_disabled);
 145 
 146     // For migrate_to and migrate_from actions, retry with "migrate"
 147     // @TODO This should be either documented or deprecated
 148     if ((action_config == NULL)
 149         && pcmk__str_any_of(action_name, PCMK_ACTION_MIGRATE_TO,
 150                             PCMK_ACTION_MIGRATE_FROM, NULL)) {
 151         action_config = find_exact_action_config(rsc, "migrate", 0,
 152                                                  include_disabled);
 153     }
 154 
 155     return action_config;
 156 }
 157 
 158 /*!
 159  * \internal
 160  * \brief Create a new action object
 161  *
 162  * \param[in]     key        Action key
 163  * \param[in]     task       Action name
 164  * \param[in,out] rsc        Resource that action is for (if any)
 165  * \param[in]     node       Node that action is on (if any)
 166  * \param[in]     optional   Whether action should be considered optional
 167  * \param[in,out] scheduler  Scheduler data
 168  *
 169  * \return Newly allocated action
 170  * \note This function takes ownership of \p key. It is the caller's
 171  *       responsibility to free the return value with pe_free_action().
 172  */
 173 static pcmk_action_t *
 174 new_action(char *key, const char *task, pcmk_resource_t *rsc,
     /*  */
 175            const pcmk_node_t *node, bool optional, pcmk_scheduler_t *scheduler)
 176 {
 177     pcmk_action_t *action = pcmk__assert_alloc(1, sizeof(pcmk_action_t));
 178 
 179     action->rsc = rsc;
 180     action->task = pcmk__str_copy(task);
 181     action->uuid = key;
 182     action->scheduler = scheduler;
 183 
 184     if (node) {
 185         action->node = pe__copy_node(node);
 186     }
 187 
 188     if (pcmk__str_eq(task, PCMK_ACTION_LRM_DELETE, pcmk__str_casei)) {
 189         // Resource history deletion for a node can be done on the DC
 190         pcmk__set_action_flags(action, pcmk__action_on_dc);
 191     }
 192 
 193     pcmk__set_action_flags(action, pcmk__action_runnable);
 194     if (optional) {
 195         pcmk__set_action_flags(action, pcmk__action_optional);
 196     } else {
 197         pcmk__clear_action_flags(action, pcmk__action_optional);
 198     }
 199 
 200     if (rsc == NULL) {
 201         action->meta = pcmk__strkey_table(free, free);
 202     } else {
 203         guint interval_ms = 0;
 204 
 205         parse_op_key(key, NULL, NULL, &interval_ms);
 206         action->op_entry = pcmk__find_action_config(rsc, task, interval_ms,
 207                                                     true);
 208 
 209         /* If the given key is for one of the many notification pseudo-actions
 210          * (pre_notify_promote, etc.), the actual action name is "notify"
 211          */
 212         if ((action->op_entry == NULL) && (strstr(key, "_notify_") != NULL)) {
 213             action->op_entry = find_exact_action_config(rsc, PCMK_ACTION_NOTIFY,
 214                                                         0, true);
 215         }
 216 
 217         unpack_operation(action, action->op_entry, interval_ms);
 218     }
 219 
 220     pcmk__rsc_trace(rsc, "Created %s action %d (%s): %s for %s on %s",
 221                     (optional? "optional" : "required"),
 222                     scheduler->priv->next_action_id, key, task,
 223                     ((rsc == NULL)? "no resource" : rsc->id),
 224                     pcmk__node_name(node));
 225     action->id = scheduler->priv->next_action_id++;
 226 
 227     scheduler->priv->actions = g_list_prepend(scheduler->priv->actions, action);
 228     if (rsc == NULL) {
 229         add_singleton(scheduler, action);
 230     } else {
 231         rsc->priv->actions = g_list_prepend(rsc->priv->actions, action);
 232     }
 233     return action;
 234 }
 235 
 236 /*!
 237  * \internal
 238  * \brief Unpack a resource's action-specific instance parameters
 239  *
 240  * \param[in]     action_xml  XML of action's configuration in CIB (if any)
 241  * \param[in,out] node_attrs  Table of node attributes (for rule evaluation)
 242  * \param[in,out] scheduler   Cluster working set (for rule evaluation)
 243  *
 244  * \return Newly allocated hash table of action-specific instance parameters
 245  */
 246 GHashTable *
 247 pcmk__unpack_action_rsc_params(const xmlNode *action_xml,
     /*  */
 248                                GHashTable *node_attrs,
 249                                pcmk_scheduler_t *scheduler)
 250 {
 251     GHashTable *params = pcmk__strkey_table(free, free);
 252 
 253     pe_rule_eval_data_t rule_data = {
 254         .node_hash = node_attrs,
 255         .now = scheduler->priv->now,
 256         .match_data = NULL,
 257         .rsc_data = NULL,
 258         .op_data = NULL
 259     };
 260 
 261     pe__unpack_dataset_nvpairs(action_xml, PCMK_XE_INSTANCE_ATTRIBUTES,
 262                                &rule_data, params, NULL, scheduler);
 263     return params;
 264 }
 265 
 266 /*!
 267  * \internal
 268  * \brief Update an action's optional flag
 269  *
 270  * \param[in,out] action    Action to update
 271  * \param[in]     optional  Requested optional status
 272  */
 273 static void
 274 update_action_optional(pcmk_action_t *action, gboolean optional)
     /*  */
 275 {
 276     // Force a non-recurring action to be optional if its resource is unmanaged
 277     if ((action->rsc != NULL) && (action->node != NULL)
 278         && !pcmk_is_set(action->flags, pcmk__action_pseudo)
 279         && !pcmk_is_set(action->rsc->flags, pcmk__rsc_managed)
 280         && (g_hash_table_lookup(action->meta, PCMK_META_INTERVAL) == NULL)) {
 281             pcmk__rsc_debug(action->rsc,
 282                             "%s on %s is optional (%s is unmanaged)",
 283                             action->uuid, pcmk__node_name(action->node),
 284                             action->rsc->id);
 285             pcmk__set_action_flags(action, pcmk__action_optional);
 286             // We shouldn't clear runnable here because ... something
 287 
 288     // Otherwise require the action if requested
 289     } else if (!optional) {
 290         pcmk__clear_action_flags(action, pcmk__action_optional);
 291     }
 292 }
 293 
 294 static enum pe_quorum_policy
 295 effective_quorum_policy(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
     /*  */
 296 {
 297     enum pe_quorum_policy policy = scheduler->no_quorum_policy;
 298 
 299     if (pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
 300         policy = pcmk_no_quorum_ignore;
 301 
 302     } else if (scheduler->no_quorum_policy == pcmk_no_quorum_demote) {
 303         switch (rsc->priv->orig_role) {
 304             case pcmk_role_promoted:
 305             case pcmk_role_unpromoted:
 306                 if (rsc->priv->next_role > pcmk_role_unpromoted) {
 307                     pe__set_next_role(rsc, pcmk_role_unpromoted,
 308                                       PCMK_OPT_NO_QUORUM_POLICY "=demote");
 309                 }
 310                 policy = pcmk_no_quorum_ignore;
 311                 break;
 312             default:
 313                 policy = pcmk_no_quorum_stop;
 314                 break;
 315         }
 316     }
 317     return policy;
 318 }
 319 
 320 /*!
 321  * \internal
 322  * \brief Update a resource action's runnable flag
 323  *
 324  * \param[in,out] action     Action to update
 325  * \param[in,out] scheduler  Scheduler data
 326  *
 327  * \note This may also schedule fencing if a stop is unrunnable.
 328  */
 329 static void
 330 update_resource_action_runnable(pcmk_action_t *action,
     /*  */
 331                                 pcmk_scheduler_t *scheduler)
 332 {
 333     pcmk_resource_t *rsc = action->rsc;
 334 
 335     if (pcmk_is_set(action->flags, pcmk__action_pseudo)) {
 336         return;
 337     }
 338 
 339     if (action->node == NULL) {
 340         pcmk__rsc_trace(rsc, "%s is unrunnable (unallocated)", action->uuid);
 341         pcmk__clear_action_flags(action, pcmk__action_runnable);
 342 
 343     } else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
 344                && !(action->node->details->online)
 345                && (!pcmk__is_guest_or_bundle_node(action->node)
 346                    || pcmk_is_set(action->node->priv->flags,
 347                                   pcmk__node_remote_reset))) {
 348         pcmk__clear_action_flags(action, pcmk__action_runnable);
 349         do_crm_log(LOG_WARNING, "%s on %s is unrunnable (node is offline)",
 350                    action->uuid, pcmk__node_name(action->node));
 351         if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)
 352             && pcmk__str_eq(action->task, PCMK_ACTION_STOP, pcmk__str_casei)
 353             && !(action->node->details->unclean)) {
 354             pe_fence_node(scheduler, action->node, "stop is unrunnable", false);
 355         }
 356 
 357     } else if (!pcmk_is_set(action->flags, pcmk__action_on_dc)
 358                && action->node->details->pending) {
 359         pcmk__clear_action_flags(action, pcmk__action_runnable);
 360         do_crm_log(LOG_WARNING,
 361                    "Action %s on %s is unrunnable (node is pending)",
 362                    action->uuid, pcmk__node_name(action->node));
 363 
 364     } else if (action->needs == pcmk__requires_nothing) {
 365         pe_action_set_reason(action, NULL, TRUE);
 366         if (pcmk__is_guest_or_bundle_node(action->node)
 367             && !pe_can_fence(scheduler, action->node)) {
 368             /* An action that requires nothing usually does not require any
 369              * fencing in order to be runnable. However, there is an exception:
 370              * such an action cannot be completed if it is on a guest node whose
 371              * host is unclean and cannot be fenced.
 372              */
 373             pcmk__rsc_debug(rsc,
 374                             "%s on %s is unrunnable "
 375                             "(node's host cannot be fenced)",
 376                             action->uuid, pcmk__node_name(action->node));
 377             pcmk__clear_action_flags(action, pcmk__action_runnable);
 378         } else {
 379             pcmk__rsc_trace(rsc,
 380                             "%s on %s does not require fencing or quorum",
 381                             action->uuid, pcmk__node_name(action->node));
 382             pcmk__set_action_flags(action, pcmk__action_runnable);
 383         }
 384 
 385     } else {
 386         switch (effective_quorum_policy(rsc, scheduler)) {
 387             case pcmk_no_quorum_stop:
 388                 pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
 389                                 action->uuid, pcmk__node_name(action->node));
 390                 pcmk__clear_action_flags(action, pcmk__action_runnable);
 391                 pe_action_set_reason(action, "no quorum", true);
 392                 break;
 393 
 394             case pcmk_no_quorum_freeze:
 395                 if (!rsc->priv->fns->active(rsc, TRUE)
 396                     || (rsc->priv->next_role > rsc->priv->orig_role)) {
 397                     pcmk__rsc_debug(rsc, "%s on %s is unrunnable (no quorum)",
 398                                     action->uuid,
 399                                     pcmk__node_name(action->node));
 400                     pcmk__clear_action_flags(action, pcmk__action_runnable);
 401                     pe_action_set_reason(action, "quorum freeze", true);
 402                 }
 403                 break;
 404 
 405             default:
 406                 //pe_action_set_reason(action, NULL, TRUE);
 407                 pcmk__set_action_flags(action, pcmk__action_runnable);
 408                 break;
 409         }
 410     }
 411 }
 412 
 413 static bool
 414 valid_stop_on_fail(const char *value)
     /*  */
 415 {
 416     return !pcmk__strcase_any_of(value,
 417                                  PCMK_VALUE_STANDBY, PCMK_VALUE_DEMOTE,
 418                                  PCMK_VALUE_STOP, NULL);
 419 }
 420 
 421 /*!
 422  * \internal
 423  * \brief Validate (and possibly reset) resource action's on_fail meta-attribute
 424  *
 425  * \param[in]     rsc            Resource that action is for
 426  * \param[in]     action_name    Action name
 427  * \param[in]     action_config  Action configuration XML from CIB (if any)
 428  * \param[in,out] meta           Table of action meta-attributes
 429  */
 430 static void
 431 validate_on_fail(const pcmk_resource_t *rsc, const char *action_name,
     /*  */
 432                  const xmlNode *action_config, GHashTable *meta)
 433 {
 434     const char *name = NULL;
 435     const char *role = NULL;
 436     const char *interval_spec = NULL;
 437     const char *value = g_hash_table_lookup(meta, PCMK_META_ON_FAIL);
 438     guint interval_ms = 0U;
 439 
 440     // Stop actions can only use certain on-fail values
 441     if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)
 442         && !valid_stop_on_fail(value)) {
 443 
 444         pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s stop "
 445                          "action to default value because '%s' is not "
 446                          "allowed for stop", rsc->id, value);
 447         g_hash_table_remove(meta, PCMK_META_ON_FAIL);
 448         return;
 449     }
 450 
 451     /* Demote actions default on-fail to the on-fail value for the first
 452      * recurring monitor for the promoted role (if any).
 453      */
 454     if (pcmk__str_eq(action_name, PCMK_ACTION_DEMOTE, pcmk__str_none)
 455         && (value == NULL)) {
 456 
 457         /* @TODO This does not consider promote options set in a meta-attribute
 458          * block (which may have rules that need to be evaluated) rather than
 459          * XML properties.
 460          */
 461         for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
 462                                                        PCMK_XE_OP, NULL, NULL);
 463              operation != NULL;
 464              operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
 465 
 466             bool enabled = false;
 467             const char *promote_on_fail = NULL;
 468 
 469             /* We only care about explicit on-fail (if promote uses default, so
 470              * can demote)
 471              */
 472             promote_on_fail = crm_element_value(operation, PCMK_META_ON_FAIL);
 473             if (promote_on_fail == NULL) {
 474                 continue;
 475             }
 476 
 477             // We only care about recurring monitors for the promoted role
 478             name = crm_element_value(operation, PCMK_XA_NAME);
 479             role = crm_element_value(operation, PCMK_XA_ROLE);
 480             if (!pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
 481                 || !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
 482                                          PCMK__ROLE_PROMOTED_LEGACY, NULL)) {
 483                 continue;
 484             }
 485             interval_spec = crm_element_value(operation, PCMK_META_INTERVAL);
 486             pcmk_parse_interval_spec(interval_spec, &interval_ms);
 487             if (interval_ms == 0U) {
 488                 continue;
 489             }
 490 
 491             // We only care about enabled monitors
 492             if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
 493                                         &enabled) == pcmk_rc_ok) && !enabled) {
 494                 continue;
 495             }
 496 
 497             /* Demote actions can't default to
 498              * PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE
 499              */
 500             if (pcmk__str_eq(promote_on_fail, PCMK_VALUE_DEMOTE,
 501                              pcmk__str_casei)) {
 502                 continue;
 503             }
 504 
 505             // Use value from first applicable promote action found
 506             pcmk__insert_dup(meta, PCMK_META_ON_FAIL, promote_on_fail);
 507         }
 508         return;
 509     }
 510 
 511     if (pcmk__str_eq(action_name, PCMK_ACTION_LRM_DELETE, pcmk__str_none)
 512         && !pcmk__str_eq(value, PCMK_VALUE_IGNORE, pcmk__str_casei)) {
 513 
 514         pcmk__insert_dup(meta, PCMK_META_ON_FAIL, PCMK_VALUE_IGNORE);
 515         return;
 516     }
 517 
 518     // PCMK_META_ON_FAIL=PCMK_VALUE_DEMOTE is allowed only for certain actions
 519     if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
 520         name = crm_element_value(action_config, PCMK_XA_NAME);
 521         role = crm_element_value(action_config, PCMK_XA_ROLE);
 522         interval_spec = crm_element_value(action_config, PCMK_META_INTERVAL);
 523         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 524 
 525         if (!pcmk__str_eq(name, PCMK_ACTION_PROMOTE, pcmk__str_none)
 526             && ((interval_ms == 0U)
 527                 || !pcmk__str_eq(name, PCMK_ACTION_MONITOR, pcmk__str_none)
 528                 || !pcmk__strcase_any_of(role, PCMK_ROLE_PROMOTED,
 529                                          PCMK__ROLE_PROMOTED_LEGACY, NULL))) {
 530 
 531             pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for %s %s "
 532                              "action to default value because 'demote' is not "
 533                              "allowed for it", rsc->id, name);
 534             g_hash_table_remove(meta, PCMK_META_ON_FAIL);
 535             return;
 536         }
 537     }
 538 }
 539 
 540 static int
 541 unpack_timeout(const char *value)
     /*  */
 542 {
 543     long long timeout_ms = crm_get_msec(value);
 544 
 545     if (timeout_ms <= 0) {
 546         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
 547     }
 548     return (int) QB_MIN(timeout_ms, INT_MAX);
 549 }
 550 
 551 // true if value contains valid, non-NULL interval origin for recurring op
 552 static bool
 553 unpack_interval_origin(const char *value, const xmlNode *xml_obj,
     /*  */
 554                        guint interval_ms, const crm_time_t *now,
 555                        long long *start_delay)
 556 {
 557     long long result = 0;
 558     guint interval_sec = pcmk__timeout_ms2s(interval_ms);
 559     crm_time_t *origin = NULL;
 560 
 561     // Ignore unspecified values and non-recurring operations
 562     if ((value == NULL) || (interval_ms == 0) || (now == NULL)) {
 563         return false;
 564     }
 565 
 566     // Parse interval origin from text
 567     origin = crm_time_new(value);
 568     if (origin == NULL) {
 569         pcmk__config_err("Ignoring '" PCMK_META_INTERVAL_ORIGIN "' for "
 570                          "operation '%s' because '%s' is not valid",
 571                          pcmk__s(pcmk__xe_id(xml_obj), "(missing ID)"), value);
 572         return false;
 573     }
 574 
 575     // Get seconds since origin (negative if origin is in the future)
 576     result = crm_time_get_seconds(now) - crm_time_get_seconds(origin);
 577     crm_time_free(origin);
 578 
 579     // Calculate seconds from closest interval to now
 580     result = result % interval_sec;
 581 
 582     // Calculate seconds remaining until next interval
 583     result = ((result <= 0)? 0 : interval_sec) - result;
 584     crm_info("Calculated a start delay of %llds for operation '%s'",
 585              result, pcmk__s(pcmk__xe_id(xml_obj), "(unspecified)"));
 586 
 587     if (start_delay != NULL) {
 588         *start_delay = result * 1000; // milliseconds
 589     }
 590     return true;
 591 }
 592 
 593 static int
 594 unpack_start_delay(const char *value, GHashTable *meta)
     /*  */
 595 {
 596     long long start_delay_ms = 0;
 597 
 598     if (value == NULL) {
 599         return 0;
 600     }
 601 
 602     start_delay_ms = crm_get_msec(value);
 603     start_delay_ms = QB_MIN(start_delay_ms, INT_MAX);
 604     if (start_delay_ms < 0) {
 605         start_delay_ms = 0;
 606     }
 607 
 608     if (meta != NULL) {
 609         g_hash_table_replace(meta, strdup(PCMK_META_START_DELAY),
 610                              pcmk__itoa(start_delay_ms));
 611     }
 612 
 613     return (int) start_delay_ms;
 614 }
 615 
 616 /*!
 617  * \internal
 618  * \brief Find a resource's most frequent recurring monitor
 619  *
 620  * \param[in] rsc  Resource to check
 621  *
 622  * \return Operation XML configured for most frequent recurring monitor for
 623  *         \p rsc (if any)
 624  */
 625 static xmlNode *
 626 most_frequent_monitor(const pcmk_resource_t *rsc)
     /*  */
 627 {
 628     guint min_interval_ms = G_MAXUINT;
 629     xmlNode *op = NULL;
 630 
 631     for (xmlNode *operation = pcmk__xe_first_child(rsc->priv->ops_xml,
 632                                                    PCMK_XE_OP, NULL, NULL);
 633          operation != NULL; operation = pcmk__xe_next(operation, PCMK_XE_OP)) {
 634 
 635         bool enabled = false;
 636         guint interval_ms = 0U;
 637         const char *interval_spec = crm_element_value(operation,
 638                                                       PCMK_META_INTERVAL);
 639 
 640         // We only care about enabled recurring monitors
 641         if (!pcmk__str_eq(crm_element_value(operation, PCMK_XA_NAME),
 642                           PCMK_ACTION_MONITOR, pcmk__str_none)) {
 643             continue;
 644         }
 645 
 646         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 647         if (interval_ms == 0U) {
 648             continue;
 649         }
 650 
 651         // @TODO This does not consider meta-attributes, rules, defaults, etc.
 652         if ((pcmk__xe_get_bool_attr(operation, PCMK_META_ENABLED,
 653                                     &enabled) == pcmk_rc_ok) && !enabled) {
 654             continue;
 655         }
 656 
 657         if (interval_ms < min_interval_ms) {
 658             min_interval_ms = interval_ms;
 659             op = operation;
 660         }
 661     }
 662     return op;
 663 }
 664 
 665 /*!
 666  * \internal
 667  * \brief Unpack action meta-attributes
 668  *
 669  * \param[in,out] rsc            Resource that action is for
 670  * \param[in]     node           Node that action is on
 671  * \param[in]     action_name    Action name
 672  * \param[in]     interval_ms    Action interval (in milliseconds)
 673  * \param[in]     action_config  Action XML configuration from CIB (if any)
 674  *
 675  * Unpack a resource action's meta-attributes (normalizing the interval,
 676  * timeout, and start delay values as integer milliseconds) from its CIB XML
 677  * configuration (including defaults).
 678  *
 679  * \return Newly allocated hash table with normalized action meta-attributes
 680  */
 681 GHashTable *
 682 pcmk__unpack_action_meta(pcmk_resource_t *rsc, const pcmk_node_t *node,
     /*  */
 683                          const char *action_name, guint interval_ms,
 684                          const xmlNode *action_config)
 685 {
 686     GHashTable *meta = NULL;
 687     const char *timeout_spec = NULL;
 688     const char *str = NULL;
 689 
 690     pe_rsc_eval_data_t rsc_rule_data = {
 691         .standard = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS),
 692         .provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER),
 693         .agent = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE),
 694     };
 695 
 696     pe_op_eval_data_t op_rule_data = {
 697         .op_name = action_name,
 698         .interval = interval_ms,
 699     };
 700 
 701     pe_rule_eval_data_t rule_data = {
 702         /* Node attributes are not set because node expressions are not allowed
 703          * for meta-attributes
 704          */
 705         .now = rsc->priv->scheduler->priv->now,
 706         .match_data = NULL,
 707         .rsc_data = &rsc_rule_data,
 708         .op_data = &op_rule_data,
 709     };
 710 
 711     meta = pcmk__strkey_table(free, free);
 712 
 713     if (action_config != NULL) {
 714         // <op> <meta_attributes> take precedence over defaults
 715         pe__unpack_dataset_nvpairs(action_config, PCMK_XE_META_ATTRIBUTES,
 716                                    &rule_data, meta, NULL,
 717                                    rsc->priv->scheduler);
 718 
 719         /* Anything set as an <op> XML property has highest precedence.
 720          * This ensures we use the name and interval from the <op> tag.
 721          * (See below for the only exception, fence device start/probe timeout.)
 722          */
 723         for (xmlAttrPtr attr = action_config->properties;
 724              attr != NULL; attr = attr->next) {
 725             pcmk__insert_dup(meta, (const char *) attr->name,
 726                              pcmk__xml_attr_value(attr));
 727         }
 728     }
 729 
 730     // Derive default timeout for probes from recurring monitor timeouts
 731     if (pcmk_is_probe(action_name, interval_ms)
 732         && (g_hash_table_lookup(meta, PCMK_META_TIMEOUT) == NULL)) {
 733 
 734         xmlNode *min_interval_mon = most_frequent_monitor(rsc);
 735 
 736         if (min_interval_mon != NULL) {
 737             /* @TODO This does not consider timeouts set in
 738              * PCMK_XE_META_ATTRIBUTES blocks (which may also have rules that
 739              * need to be evaluated).
 740              */
 741             timeout_spec = crm_element_value(min_interval_mon,
 742                                              PCMK_META_TIMEOUT);
 743             if (timeout_spec != NULL) {
 744                 pcmk__rsc_trace(rsc,
 745                                 "Setting default timeout for %s probe to "
 746                                 "most frequent monitor's timeout '%s'",
 747                                 rsc->id, timeout_spec);
 748                 pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
 749             }
 750         }
 751     }
 752 
 753     // Cluster-wide <op_defaults> <meta_attributes>
 754     pe__unpack_dataset_nvpairs(rsc->priv->scheduler->priv->op_defaults,
 755                                PCMK_XE_META_ATTRIBUTES, &rule_data, meta, NULL,
 756                                rsc->priv->scheduler);
 757 
 758     g_hash_table_remove(meta, PCMK_XA_ID);
 759 
 760     // Normalize interval to milliseconds
 761     if (interval_ms > 0) {
 762         g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_INTERVAL),
 763                             crm_strdup_printf("%u", interval_ms));
 764     } else {
 765         g_hash_table_remove(meta, PCMK_META_INTERVAL);
 766     }
 767 
 768     /* Timeout order of precedence (highest to lowest):
 769      *   1. pcmk_monitor_timeout resource parameter (only for starts and probes
 770      *      when rsc has pcmk_ra_cap_fence_params; this gets used for recurring
 771      *      monitors via the executor instead)
 772      *   2. timeout configured in <op> (with <op timeout> taking precedence over
 773      *      <op> <meta_attributes>)
 774      *   3. timeout configured in <op_defaults> <meta_attributes>
 775      *   4. PCMK_DEFAULT_ACTION_TIMEOUT_MS
 776      */
 777 
 778     // Check for pcmk_monitor_timeout
 779     if (pcmk_is_set(pcmk_get_ra_caps(rsc_rule_data.standard),
 780                     pcmk_ra_cap_fence_params)
 781         && (pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)
 782             || pcmk_is_probe(action_name, interval_ms))) {
 783 
 784         GHashTable *params = pe_rsc_params(rsc, node, rsc->priv->scheduler);
 785 
 786         timeout_spec = g_hash_table_lookup(params, "pcmk_monitor_timeout");
 787         if (timeout_spec != NULL) {
 788             pcmk__rsc_trace(rsc,
 789                             "Setting timeout for %s %s to "
 790                             "pcmk_monitor_timeout (%s)",
 791                             rsc->id, action_name, timeout_spec);
 792             pcmk__insert_dup(meta, PCMK_META_TIMEOUT, timeout_spec);
 793         }
 794     }
 795 
 796     // Normalize timeout to positive milliseconds
 797     timeout_spec = g_hash_table_lookup(meta, PCMK_META_TIMEOUT);
 798     g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_TIMEOUT),
 799                         pcmk__itoa(unpack_timeout(timeout_spec)));
 800 
 801     // Ensure on-fail has a valid value
 802     validate_on_fail(rsc, action_name, action_config, meta);
 803 
 804     // Normalize PCMK_META_START_DELAY
 805     str = g_hash_table_lookup(meta, PCMK_META_START_DELAY);
 806     if (str != NULL) {
 807         unpack_start_delay(str, meta);
 808     } else {
 809         long long start_delay = 0;
 810 
 811         str = g_hash_table_lookup(meta, PCMK_META_INTERVAL_ORIGIN);
 812         if (unpack_interval_origin(str, action_config, interval_ms,
 813                                    rsc->priv->scheduler->priv->now,
 814                                    &start_delay)) {
 815             g_hash_table_insert(meta, pcmk__str_copy(PCMK_META_START_DELAY),
 816                                 crm_strdup_printf("%lld", start_delay));
 817         }
 818     }
 819     return meta;
 820 }
 821 
 822 /*!
 823  * \internal
 824  * \brief Determine an action's quorum and fencing dependency
 825  *
 826  * \param[in] rsc          Resource that action is for
 827  * \param[in] action_name  Name of action being unpacked
 828  *
 829  * \return Quorum and fencing dependency appropriate to action
 830  */
 831 enum pcmk__requires
 832 pcmk__action_requires(const pcmk_resource_t *rsc, const char *action_name)
     /*  */
 833 {
 834     const char *value = NULL;
 835     enum pcmk__requires requires = pcmk__requires_nothing;
 836 
 837     CRM_CHECK((rsc != NULL) && (action_name != NULL), return requires);
 838 
 839     if (!pcmk__strcase_any_of(action_name, PCMK_ACTION_START,
 840                               PCMK_ACTION_PROMOTE, NULL)) {
 841         value = "nothing (not start or promote)";
 842 
 843     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
 844         requires = pcmk__requires_fencing;
 845         value = "fencing";
 846 
 847     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_quorum)) {
 848         requires = pcmk__requires_quorum;
 849         value = "quorum";
 850 
 851     } else {
 852         value = "nothing";
 853     }
 854     pcmk__rsc_trace(rsc, "%s of %s requires %s", action_name, rsc->id, value);
 855     return requires;
 856 }
 857 
 858 /*!
 859  * \internal
 860  * \brief Parse action failure response from a user-provided string
 861  *
 862  * \param[in] rsc          Resource that action is for
 863  * \param[in] action_name  Name of action
 864  * \param[in] interval_ms  Action interval (in milliseconds)
 865  * \param[in] value        User-provided configuration value for on-fail
 866  *
 867  * \return Action failure response parsed from \p text
 868  */
 869 enum pcmk__on_fail
 870 pcmk__parse_on_fail(const pcmk_resource_t *rsc, const char *action_name,
     /*  */
 871                     guint interval_ms, const char *value)
 872 {
 873     const char *desc = NULL;
 874     bool needs_remote_reset = false;
 875     enum pcmk__on_fail on_fail = pcmk__on_fail_ignore;
 876     const pcmk_scheduler_t *scheduler = NULL;
 877 
 878     // There's no enum value for unknown or invalid, so assert
 879     pcmk__assert((rsc != NULL) && (action_name != NULL));
 880     scheduler = rsc->priv->scheduler;
 881 
 882     if (value == NULL) {
 883         // Use default
 884 
 885     } else if (pcmk__str_eq(value, PCMK_VALUE_BLOCK, pcmk__str_casei)) {
 886         on_fail = pcmk__on_fail_block;
 887         desc = "block";
 888 
 889     } else if (pcmk__str_eq(value, PCMK_VALUE_FENCE, pcmk__str_casei)) {
 890         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
 891             on_fail = pcmk__on_fail_fence_node;
 892             desc = "node fencing";
 893         } else {
 894             pcmk__config_err("Resetting '" PCMK_META_ON_FAIL "' for "
 895                              "%s of %s to 'stop' because 'fence' is not "
 896                              "valid when fencing is disabled",
 897                              action_name, rsc->id);
 898             on_fail = pcmk__on_fail_stop;
 899             desc = "stop resource";
 900         }
 901 
 902     } else if (pcmk__str_eq(value, PCMK_VALUE_STANDBY, pcmk__str_casei)) {
 903         on_fail = pcmk__on_fail_standby_node;
 904         desc = "node standby";
 905 
 906     } else if (pcmk__strcase_any_of(value,
 907                                     PCMK_VALUE_IGNORE, PCMK_VALUE_NOTHING,
 908                                     NULL)) {
 909         desc = "ignore";
 910 
 911     } else if (pcmk__str_eq(value, "migrate", pcmk__str_casei)) {
 912         on_fail = pcmk__on_fail_ban;
 913         desc = "force migration";
 914 
 915     } else if (pcmk__str_eq(value, PCMK_VALUE_STOP, pcmk__str_casei)) {
 916         on_fail = pcmk__on_fail_stop;
 917         desc = "stop resource";
 918 
 919     } else if (pcmk__str_eq(value, PCMK_VALUE_RESTART, pcmk__str_casei)) {
 920         on_fail = pcmk__on_fail_restart;
 921         desc = "restart (and possibly migrate)";
 922 
 923     } else if (pcmk__str_eq(value, PCMK_VALUE_RESTART_CONTAINER,
 924                             pcmk__str_casei)) {
 925         if (rsc->priv->launcher == NULL) {
 926             pcmk__rsc_debug(rsc,
 927                             "Using default " PCMK_META_ON_FAIL " for %s "
 928                             "of %s because it does not have a launcher",
 929                             action_name, rsc->id);
 930         } else {
 931             on_fail = pcmk__on_fail_restart_container;
 932             desc = "restart container (and possibly migrate)";
 933         }
 934 
 935     } else if (pcmk__str_eq(value, PCMK_VALUE_DEMOTE, pcmk__str_casei)) {
 936         on_fail = pcmk__on_fail_demote;
 937         desc = "demote instance";
 938 
 939     } else {
 940         pcmk__config_err("Using default '" PCMK_META_ON_FAIL "' for "
 941                          "%s of %s because '%s' is not valid",
 942                          action_name, rsc->id, value);
 943     }
 944 
 945     /* Remote node connections are handled specially. Failures that result
 946      * in dropping an active connection must result in fencing. The only
 947      * failures that don't are probes and starts. The user can explicitly set
 948      * PCMK_META_ON_FAIL=PCMK_VALUE_FENCE to fence after start failures.
 949      */
 950     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
 951         && pcmk__is_remote_node(pcmk_find_node(scheduler, rsc->id))
 952         && !pcmk_is_probe(action_name, interval_ms)
 953         && !pcmk__str_eq(action_name, PCMK_ACTION_START, pcmk__str_none)) {
 954         needs_remote_reset = true;
 955         if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 956             desc = NULL; // Force default for unmanaged connections
 957         }
 958     }
 959 
 960     if (desc != NULL) {
 961         // Explicit value used, default not needed
 962 
 963     } else if (rsc->priv->launcher != NULL) {
 964         on_fail = pcmk__on_fail_restart_container;
 965         desc = "restart container (and possibly migrate) (default)";
 966 
 967     } else if (needs_remote_reset) {
 968         if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 969             if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
 970                 desc = "fence remote node (default)";
 971             } else {
 972                 desc = "recover remote node connection (default)";
 973             }
 974             on_fail = pcmk__on_fail_reset_remote;
 975         } else {
 976             on_fail = pcmk__on_fail_stop;
 977             desc = "stop unmanaged remote node (enforcing default)";
 978         }
 979 
 980     } else if (pcmk__str_eq(action_name, PCMK_ACTION_STOP, pcmk__str_none)) {
 981         if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
 982             on_fail = pcmk__on_fail_fence_node;
 983             desc = "resource fence (default)";
 984         } else {
 985             on_fail = pcmk__on_fail_block;
 986             desc = "resource block (default)";
 987         }
 988 
 989     } else {
 990         on_fail = pcmk__on_fail_restart;
 991         desc = "restart (and possibly migrate) (default)";
 992     }
 993 
 994     pcmk__rsc_trace(rsc, "Failure handling for %s-interval %s of %s: %s",
 995                     pcmk__readable_interval(interval_ms), action_name,
 996                     rsc->id, desc);
 997     return on_fail;
 998 }
 999 
1000 /*!
1001  * \internal
1002  * \brief Determine a resource's role after failure of an action
1003  *
1004  * \param[in] rsc          Resource that action is for
1005  * \param[in] action_name  Action name
1006  * \param[in] on_fail      Failure handling for action
1007  * \param[in] meta         Unpacked action meta-attributes
1008  *
1009  * \return Resource role that results from failure of action
1010  */
1011 enum rsc_role_e
1012 pcmk__role_after_failure(const pcmk_resource_t *rsc, const char *action_name,
     /*  */
1013                          enum pcmk__on_fail on_fail, GHashTable *meta)
1014 {
1015     enum rsc_role_e role = pcmk_role_unknown;
1016 
1017     // Set default for role after failure specially in certain circumstances
1018     switch (on_fail) {
1019         case pcmk__on_fail_stop:
1020             role = pcmk_role_stopped;
1021             break;
1022 
1023         case pcmk__on_fail_reset_remote:
1024             if (rsc->priv->remote_reconnect_ms != 0U) {
1025                 role = pcmk_role_stopped;
1026             }
1027             break;
1028 
1029         default:
1030             break;
1031     }
1032 
1033     if (role == pcmk_role_unknown) {
1034         // Use default
1035         if (pcmk__str_eq(action_name, PCMK_ACTION_PROMOTE, pcmk__str_none)) {
1036             role = pcmk_role_unpromoted;
1037         } else {
1038             role = pcmk_role_started;
1039         }
1040     }
1041     pcmk__rsc_trace(rsc, "Role after %s %s failure is: %s",
1042                     rsc->id, action_name, pcmk_role_text(role));
1043     return role;
1044 }
1045 
1046 /*!
1047  * \internal
1048  * \brief Unpack action configuration
1049  *
1050  * Unpack a resource action's meta-attributes (normalizing the interval,
1051  * timeout, and start delay values as integer milliseconds), requirements, and
1052  * failure policy from its CIB XML configuration (including defaults).
1053  *
1054  * \param[in,out] action       Resource action to unpack into
1055  * \param[in]     xml_obj      Action configuration XML (NULL for defaults only)
1056  * \param[in]     interval_ms  How frequently to perform the operation
1057  */
1058 static void
1059 unpack_operation(pcmk_action_t *action, const xmlNode *xml_obj,
     /*  */
1060                  guint interval_ms)
1061 {
1062     const char *value = NULL;
1063 
1064     action->meta = pcmk__unpack_action_meta(action->rsc, action->node,
1065                                             action->task, interval_ms, xml_obj);
1066     action->needs = pcmk__action_requires(action->rsc, action->task);
1067 
1068     value = g_hash_table_lookup(action->meta, PCMK_META_ON_FAIL);
1069     action->on_fail = pcmk__parse_on_fail(action->rsc, action->task,
1070                                           interval_ms, value);
1071 
1072     action->fail_role = pcmk__role_after_failure(action->rsc, action->task,
1073                                                  action->on_fail, action->meta);
1074 }
1075 
1076 /*!
1077  * \brief Create or update an action object
1078  *
1079  * \param[in,out] rsc          Resource that action is for (if any)
1080  * \param[in,out] key          Action key (must be non-NULL)
1081  * \param[in]     task         Action name (must be non-NULL)
1082  * \param[in]     on_node      Node that action is on (if any)
1083  * \param[in]     optional     Whether action should be considered optional
1084  * \param[in,out] scheduler    Scheduler data
1085  *
1086  * \return Action object corresponding to arguments (guaranteed not to be
1087  *         \c NULL)
1088  * \note This function takes ownership of (and might free) \p key, and
1089  *       \p scheduler takes ownership of the returned action (the caller should
1090  *       not free it).
1091  */
1092 pcmk_action_t *
1093 custom_action(pcmk_resource_t *rsc, char *key, const char *task,
     /*  */
1094               const pcmk_node_t *on_node, gboolean optional,
1095               pcmk_scheduler_t *scheduler)
1096 {
1097     pcmk_action_t *action = NULL;
1098 
1099     pcmk__assert((key != NULL) && (task != NULL) && (scheduler != NULL));
1100 
1101     action = find_existing_action(key, rsc, on_node, scheduler);
1102     if (action == NULL) {
1103         action = new_action(key, task, rsc, on_node, optional, scheduler);
1104     } else {
1105         free(key);
1106     }
1107 
1108     update_action_optional(action, optional);
1109 
1110     if (rsc != NULL) {
1111         /* An action can be initially created with a NULL node, and later have
1112          * the node added via find_existing_action() (above) -> find_actions().
1113          * That is why the extra parameters are unpacked here rather than in
1114          * new_action().
1115          */
1116         if ((action->node != NULL) && (action->op_entry != NULL)
1117             && !pcmk_is_set(action->flags, pcmk__action_attrs_evaluated)) {
1118 
1119             GHashTable *attrs = action->node->priv->attrs;
1120 
1121             if (action->extra != NULL) {
1122                 g_hash_table_destroy(action->extra);
1123             }
1124             action->extra = pcmk__unpack_action_rsc_params(action->op_entry,
1125                                                            attrs, scheduler);
1126             pcmk__set_action_flags(action, pcmk__action_attrs_evaluated);
1127         }
1128 
1129         update_resource_action_runnable(action, scheduler);
1130     }
1131 
1132     if (action->extra == NULL) {
1133         action->extra = pcmk__strkey_table(free, free);
1134     }
1135 
1136     return action;
1137 }
1138 
1139 pcmk_action_t *
1140 get_pseudo_op(const char *name, pcmk_scheduler_t *scheduler)
     /*  */
1141 {
1142     pcmk_action_t *op = lookup_singleton(scheduler, name);
1143 
1144     if (op == NULL) {
1145         op = custom_action(NULL, strdup(name), name, NULL, TRUE, scheduler);
1146         pcmk__set_action_flags(op, pcmk__action_pseudo|pcmk__action_runnable);
1147     }
1148     return op;
1149 }
1150 
1151 static GList *
1152 find_unfencing_devices(GList *candidates, GList *matches) 
     /*  */
1153 {
1154     for (GList *gIter = candidates; gIter != NULL; gIter = gIter->next) {
1155         pcmk_resource_t *candidate = gIter->data;
1156 
1157         if (candidate->priv->children != NULL) {
1158             matches = find_unfencing_devices(candidate->priv->children,
1159                                              matches);
1160 
1161         } else if (!pcmk_is_set(candidate->flags, pcmk__rsc_fence_device)) {
1162             continue;
1163 
1164         } else if (pcmk_is_set(candidate->flags, pcmk__rsc_needs_unfencing)) {
1165             matches = g_list_prepend(matches, candidate);
1166 
1167         } else if (pcmk__str_eq(g_hash_table_lookup(candidate->priv->meta,
1168                                                     PCMK_STONITH_PROVIDES),
1169                                 PCMK_VALUE_UNFENCING, pcmk__str_casei)) {
1170             matches = g_list_prepend(matches, candidate);
1171         }
1172     }
1173     return matches;
1174 }
1175 
1176 static int
1177 node_priority_fencing_delay(const pcmk_node_t *node,
     /*  */
1178                             const pcmk_scheduler_t *scheduler)
1179 {
1180     int member_count = 0;
1181     int online_count = 0;
1182     int top_priority = 0;
1183     int lowest_priority = 0;
1184     GList *gIter = NULL;
1185 
1186     // PCMK_OPT_PRIORITY_FENCING_DELAY is disabled
1187     if (scheduler->priv->priority_fencing_ms == 0U) {
1188         return 0;
1189     }
1190 
1191     /* No need to request a delay if the fencing target is not a normal cluster
1192      * member, for example if it's a remote node or a guest node. */
1193     if (node->priv->variant != pcmk__node_variant_cluster) {
1194         return 0;
1195     }
1196 
1197     // No need to request a delay if the fencing target is in our partition
1198     if (node->details->online) {
1199         return 0;
1200     }
1201 
1202     for (gIter = scheduler->nodes; gIter != NULL; gIter = gIter->next) {
1203         pcmk_node_t *n = gIter->data;
1204 
1205         if (n->priv->variant != pcmk__node_variant_cluster) {
1206             continue;
1207         }
1208 
1209         member_count ++;
1210 
1211         if (n->details->online) {
1212             online_count++;
1213         }
1214 
1215         if (member_count == 1
1216             || n->priv->priority > top_priority) {
1217             top_priority = n->priv->priority;
1218         }
1219 
1220         if (member_count == 1
1221             || n->priv->priority < lowest_priority) {
1222             lowest_priority = n->priv->priority;
1223         }
1224     }
1225 
1226     // No need to delay if we have more than half of the cluster members
1227     if (online_count > member_count / 2) {
1228         return 0;
1229     }
1230 
1231     /* All the nodes have equal priority.
1232      * Any configured corresponding `pcmk_delay_base/max` will be applied. */
1233     if (lowest_priority == top_priority) {
1234         return 0;
1235     }
1236 
1237     if (node->priv->priority < top_priority) {
1238         return 0;
1239     }
1240 
1241     return pcmk__timeout_ms2s(scheduler->priv->priority_fencing_ms);
1242 }
1243 
1244 pcmk_action_t *
1245 pe_fence_op(pcmk_node_t *node, const char *op, bool optional,
     /*  */
1246             const char *reason, bool priority_delay,
1247             pcmk_scheduler_t *scheduler)
1248 {
1249     char *op_key = NULL;
1250     pcmk_action_t *stonith_op = NULL;
1251 
1252     if(op == NULL) {
1253         op = scheduler->priv->fence_action;
1254     }
1255 
1256     op_key = crm_strdup_printf("%s-%s-%s",
1257                                PCMK_ACTION_STONITH, node->priv->name, op);
1258 
1259     stonith_op = lookup_singleton(scheduler, op_key);
1260     if(stonith_op == NULL) {
1261         stonith_op = custom_action(NULL, op_key, PCMK_ACTION_STONITH, node,
1262                                    TRUE, scheduler);
1263 
1264         pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE, node->priv->name);
1265         pcmk__insert_meta(stonith_op, PCMK__META_ON_NODE_UUID,
1266                           node->priv->id);
1267         pcmk__insert_meta(stonith_op, PCMK__META_STONITH_ACTION, op);
1268 
1269         if (pcmk_is_set(scheduler->flags, pcmk__sched_enable_unfencing)) {
1270             /* Extra work to detect device changes
1271              */
1272             GString *digests_all = g_string_sized_new(1024);
1273             GString *digests_secure = g_string_sized_new(1024);
1274 
1275             GList *matches = find_unfencing_devices(scheduler->priv->resources,
1276                                                     NULL);
1277 
1278             for (GList *gIter = matches; gIter != NULL; gIter = gIter->next) {
1279                 pcmk_resource_t *match = gIter->data;
1280                 const char *agent = g_hash_table_lookup(match->priv->meta,
1281                                                         PCMK_XA_TYPE);
1282                 pcmk__op_digest_t *data = NULL;
1283 
1284                 data = pe__compare_fencing_digest(match, agent, node,
1285                                                   scheduler);
1286                 if (data->rc == pcmk__digest_mismatch) {
1287                     optional = FALSE;
1288                     crm_notice("Unfencing node %s because the definition of "
1289                                "%s changed", pcmk__node_name(node), match->id);
1290                     if (!pcmk__is_daemon && (scheduler->priv->out != NULL)) {
1291                         pcmk__output_t *out = scheduler->priv->out;
1292 
1293                         out->info(out,
1294                                   "notice: Unfencing node %s because the "
1295                                   "definition of %s changed",
1296                                   pcmk__node_name(node), match->id);
1297                     }
1298                 }
1299 
1300                 pcmk__g_strcat(digests_all,
1301                                match->id, ":", agent, ":",
1302                                data->digest_all_calc, ",", NULL);
1303                 pcmk__g_strcat(digests_secure,
1304                                match->id, ":", agent, ":",
1305                                data->digest_secure_calc, ",", NULL);
1306             }
1307             pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_ALL,
1308                              digests_all->str);
1309             g_string_free(digests_all, TRUE);
1310 
1311             pcmk__insert_dup(stonith_op->meta, PCMK__META_DIGESTS_SECURE,
1312                              digests_secure->str);
1313             g_string_free(digests_secure, TRUE);
1314 
1315             g_list_free(matches);
1316         }
1317 
1318     } else {
1319         free(op_key);
1320     }
1321 
1322     if ((scheduler->priv->priority_fencing_ms > 0U)
1323 
1324             /* It's a suitable case where PCMK_OPT_PRIORITY_FENCING_DELAY
1325              * applies. At least add PCMK_OPT_PRIORITY_FENCING_DELAY field as
1326              * an indicator.
1327              */
1328         && (priority_delay
1329 
1330             /* The priority delay needs to be recalculated if this function has
1331              * been called by schedule_fencing_and_shutdowns() after node
1332              * priority has already been calculated by native_add_running().
1333              */
1334             || g_hash_table_lookup(stonith_op->meta,
1335                                    PCMK_OPT_PRIORITY_FENCING_DELAY) != NULL)) {
1336 
1337             /* Add PCMK_OPT_PRIORITY_FENCING_DELAY to the fencing op even if
1338              * it's 0 for the targeting node. So that it takes precedence over
1339              * any possible `pcmk_delay_base/max`.
1340              */
1341             char *delay_s = pcmk__itoa(node_priority_fencing_delay(node,
1342                                                                    scheduler));
1343 
1344             g_hash_table_insert(stonith_op->meta,
1345                                 strdup(PCMK_OPT_PRIORITY_FENCING_DELAY),
1346                                 delay_s);
1347     }
1348 
1349     if(optional == FALSE && pe_can_fence(scheduler, node)) {
1350         pcmk__clear_action_flags(stonith_op, pcmk__action_optional);
1351         pe_action_set_reason(stonith_op, reason, false);
1352 
1353     } else if(reason && stonith_op->reason == NULL) {
1354         stonith_op->reason = strdup(reason);
1355     }
1356 
1357     return stonith_op;
1358 }
1359 
1360 void
1361 pe_free_action(pcmk_action_t *action)
     /*  */
1362 {
1363     if (action == NULL) {
1364         return;
1365     }
1366     g_list_free_full(action->actions_before, free);
1367     g_list_free_full(action->actions_after, free);
1368     if (action->extra) {
1369         g_hash_table_destroy(action->extra);
1370     }
1371     if (action->meta) {
1372         g_hash_table_destroy(action->meta);
1373     }
1374     pcmk__free_node_copy(action->node);
1375     free(action->cancel_task);
1376     free(action->reason);
1377     free(action->task);
1378     free(action->uuid);
1379     free(action);
1380 }
1381 
1382 enum pcmk__action_type
1383 get_complex_task(const pcmk_resource_t *rsc, const char *name)
     /*  */
1384 {
1385     enum pcmk__action_type task = pcmk__parse_action(name);
1386 
1387     if (pcmk__is_primitive(rsc)) {
1388         switch (task) {
1389             case pcmk__action_stopped:
1390             case pcmk__action_started:
1391             case pcmk__action_demoted:
1392             case pcmk__action_promoted:
1393                 crm_trace("Folding %s back into its atomic counterpart for %s",
1394                           name, rsc->id);
1395                 --task;
1396                 break;
1397             default:
1398                 break;
1399         }
1400     }
1401     return task;
1402 }
1403 
1404 /*!
1405  * \internal
1406  * \brief Find first matching action in a list
1407  *
1408  * \param[in] input    List of actions to search
1409  * \param[in] uuid     If not NULL, action must have this UUID
1410  * \param[in] task     If not NULL, action must have this action name
1411  * \param[in] on_node  If not NULL, action must be on this node
1412  *
1413  * \return First action in list that matches criteria, or NULL if none
1414  */
1415 pcmk_action_t *
1416 find_first_action(const GList *input, const char *uuid, const char *task,
     /*  */
1417                   const pcmk_node_t *on_node)
1418 {
1419     CRM_CHECK(uuid || task, return NULL);
1420 
1421     for (const GList *gIter = input; gIter != NULL; gIter = gIter->next) {
1422         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
1423 
1424         if (uuid != NULL && !pcmk__str_eq(uuid, action->uuid, pcmk__str_casei)) {
1425             continue;
1426 
1427         } else if (task != NULL && !pcmk__str_eq(task, action->task, pcmk__str_casei)) {
1428             continue;
1429 
1430         } else if (on_node == NULL) {
1431             return action;
1432 
1433         } else if (action->node == NULL) {
1434             continue;
1435 
1436         } else if (pcmk__same_node(on_node, action->node)) {
1437             return action;
1438         }
1439     }
1440 
1441     return NULL;
1442 }
1443 
1444 GList *
1445 find_actions(GList *input, const char *key, const pcmk_node_t *on_node)
     /*  */
1446 {
1447     GList *gIter = input;
1448     GList *result = NULL;
1449 
1450     CRM_CHECK(key != NULL, return NULL);
1451 
1452     for (; gIter != NULL; gIter = gIter->next) {
1453         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
1454 
1455         if (!pcmk__str_eq(key, action->uuid, pcmk__str_casei)) {
1456             continue;
1457 
1458         } else if (on_node == NULL) {
1459             crm_trace("Action %s matches (ignoring node)", key);
1460             result = g_list_prepend(result, action);
1461 
1462         } else if (action->node == NULL) {
1463             crm_trace("Action %s matches (unallocated, assigning to %s)",
1464                       key, pcmk__node_name(on_node));
1465 
1466             action->node = pe__copy_node(on_node);
1467             result = g_list_prepend(result, action);
1468 
1469         } else if (pcmk__same_node(on_node, action->node)) {
1470             crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
1471             result = g_list_prepend(result, action);
1472         }
1473     }
1474 
1475     return result;
1476 }
1477 
1478 GList *
1479 find_actions_exact(GList *input, const char *key, const pcmk_node_t *on_node)
     /*  */
1480 {
1481     GList *result = NULL;
1482 
1483     CRM_CHECK(key != NULL, return NULL);
1484 
1485     if (on_node == NULL) {
1486         return NULL;
1487     }
1488 
1489     for (GList *gIter = input; gIter != NULL; gIter = gIter->next) {
1490         pcmk_action_t *action = (pcmk_action_t *) gIter->data;
1491 
1492         if ((action->node != NULL)
1493             && pcmk__str_eq(key, action->uuid, pcmk__str_casei)
1494             && pcmk__same_node(on_node, action->node)) {
1495 
1496             crm_trace("Action %s on %s matches", key, pcmk__node_name(on_node));
1497             result = g_list_prepend(result, action);
1498         }
1499     }
1500 
1501     return result;
1502 }
1503 
1504 /*!
1505  * \brief Find all actions of given type for a resource
1506  *
1507  * \param[in] rsc           Resource to search
1508  * \param[in] node          Find only actions scheduled on this node
1509  * \param[in] task          Action name to search for
1510  * \param[in] require_node  If TRUE, NULL node or action node will not match
1511  *
1512  * \return List of actions found (or NULL if none)
1513  * \note If node is not NULL and require_node is FALSE, matching actions
1514  *       without a node will be assigned to node.
1515  */
1516 GList *
1517 pe__resource_actions(const pcmk_resource_t *rsc, const pcmk_node_t *node,
     /*  */
1518                      const char *task, bool require_node)
1519 {
1520     GList *result = NULL;
1521     char *key = pcmk__op_key(rsc->id, task, 0);
1522 
1523     if (require_node) {
1524         result = find_actions_exact(rsc->priv->actions, key, node);
1525     } else {
1526         result = find_actions(rsc->priv->actions, key, node);
1527     }
1528     free(key);
1529     return result;
1530 }
1531 
1532 /*!
1533  * \internal
1534  * \brief Create an action reason string based on the action itself
1535  *
1536  * \param[in] action  Action to create reason string for
1537  * \param[in] flag    Action flag that was cleared
1538  *
1539  * \return Newly allocated string suitable for use as action reason
1540  * \note It is the caller's responsibility to free() the result.
1541  */
1542 char *
1543 pe__action2reason(const pcmk_action_t *action, enum pcmk__action_flags flag)
     /*  */
1544 {
1545     const char *change = NULL;
1546 
1547     switch (flag) {
1548         case pcmk__action_runnable:
1549             change = "unrunnable";
1550             break;
1551         case pcmk__action_migratable:
1552             change = "unmigrateable";
1553             break;
1554         case pcmk__action_optional:
1555             change = "required";
1556             break;
1557         default:
1558             // Bug: caller passed unsupported flag
1559             CRM_CHECK(change != NULL, change = "");
1560             break;
1561     }
1562     return crm_strdup_printf("%s%s%s %s", change,
1563                              (action->rsc == NULL)? "" : " ",
1564                              (action->rsc == NULL)? "" : action->rsc->id,
1565                              action->task);
1566 }
1567 
1568 void pe_action_set_reason(pcmk_action_t *action, const char *reason,
     /*  */
1569                           bool overwrite)
1570 {
1571     if (action->reason != NULL && overwrite) {
1572         pcmk__rsc_trace(action->rsc, "Changing %s reason from '%s' to '%s'",
1573                         action->uuid, action->reason,
1574                         pcmk__s(reason, "(none)"));
1575     } else if (action->reason == NULL) {
1576         pcmk__rsc_trace(action->rsc, "Set %s reason to '%s'",
1577                         action->uuid, pcmk__s(reason, "(none)"));
1578     } else {
1579         // crm_assert(action->reason != NULL && !overwrite);
1580         return;
1581     }
1582 
1583     pcmk__str_update(&action->reason, reason);
1584 }
1585 
1586 /*!
1587  * \internal
1588  * \brief Create an action to clear a resource's history from CIB
1589  *
1590  * \param[in,out] rsc       Resource to clear
1591  * \param[in]     node      Node to clear history on
1592  */
1593 void
1594 pe__clear_resource_history(pcmk_resource_t *rsc, const pcmk_node_t *node)
     /*  */
1595 {
1596     pcmk__assert((rsc != NULL) && (node != NULL));
1597 
1598     custom_action(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
1599                   PCMK_ACTION_LRM_DELETE, node, FALSE, rsc->priv->scheduler);
1600 }
1601 
1602 #define sort_return(an_int, why) do {                                   \
1603         free(a_uuid);                                           \
1604         free(b_uuid);                                           \
1605         crm_trace("%s (%d) %c %s (%d) : %s",                            \
1606                   a_xml_id, a_call_id, an_int>0?'>':an_int<0?'<':'=',   \
1607                   b_xml_id, b_call_id, why);                            \
1608         return an_int;                                                  \
1609     } while(0)
1610 
1611 int
1612 pe__is_newer_op(const xmlNode *xml_a, const xmlNode *xml_b)
     /*  */
1613 {
1614     int a_call_id = -1;
1615     int b_call_id = -1;
1616 
1617     char *a_uuid = NULL;
1618     char *b_uuid = NULL;
1619 
1620     const char *a_xml_id = crm_element_value(xml_a, PCMK_XA_ID);
1621     const char *b_xml_id = crm_element_value(xml_b, PCMK_XA_ID);
1622 
1623     const char *a_node = crm_element_value(xml_a, PCMK__META_ON_NODE);
1624     const char *b_node = crm_element_value(xml_b, PCMK__META_ON_NODE);
1625     bool same_node = pcmk__str_eq(a_node, b_node, pcmk__str_casei);
1626 
1627     if (same_node && pcmk__str_eq(a_xml_id, b_xml_id, pcmk__str_none)) {
1628         /* We have duplicate PCMK__XE_LRM_RSC_OP entries in the status
1629          * section which is unlikely to be a good thing
1630          *    - we can handle it easily enough, but we need to get
1631          *    to the bottom of why it's happening.
1632          */
1633         pcmk__config_err("Duplicate " PCMK__XE_LRM_RSC_OP " entries named %s",
1634                          a_xml_id);
1635         sort_return(0, "duplicate");
1636     }
1637 
1638     crm_element_value_int(xml_a, PCMK__XA_CALL_ID, &a_call_id);
1639     crm_element_value_int(xml_b, PCMK__XA_CALL_ID, &b_call_id);
1640 
1641     if (a_call_id == -1 && b_call_id == -1) {
1642         /* both are pending ops so it doesn't matter since
1643          *   stops are never pending
1644          */
1645         sort_return(0, "pending");
1646 
1647     } else if (same_node && a_call_id >= 0 && a_call_id < b_call_id) {
1648         sort_return(-1, "call id");
1649 
1650     } else if (same_node && b_call_id >= 0 && a_call_id > b_call_id) {
1651         sort_return(1, "call id");
1652 
1653     } else if (a_call_id >= 0 && b_call_id >= 0
1654                && (!same_node || a_call_id == b_call_id)) {
1655         /* The op and last_failed_op are the same. Order on
1656          * PCMK_XA_LAST_RC_CHANGE.
1657          */
1658         time_t last_a = -1;
1659         time_t last_b = -1;
1660 
1661         crm_element_value_epoch(xml_a, PCMK_XA_LAST_RC_CHANGE, &last_a);
1662         crm_element_value_epoch(xml_b, PCMK_XA_LAST_RC_CHANGE, &last_b);
1663 
1664         crm_trace("rc-change: %lld vs %lld",
1665                   (long long) last_a, (long long) last_b);
1666         if (last_a >= 0 && last_a < last_b) {
1667             sort_return(-1, "rc-change");
1668 
1669         } else if (last_b >= 0 && last_a > last_b) {
1670             sort_return(1, "rc-change");
1671         }
1672         sort_return(0, "rc-change");
1673 
1674     } else {
1675         /* One of the inputs is a pending operation.
1676          * Attempt to use PCMK__XA_TRANSITION_MAGIC to determine its age relative
1677          * to the other.
1678          */
1679 
1680         int a_id = -1;
1681         int b_id = -1;
1682 
1683         const char *a_magic = crm_element_value(xml_a,
1684                                                 PCMK__XA_TRANSITION_MAGIC);
1685         const char *b_magic = crm_element_value(xml_b,
1686                                                 PCMK__XA_TRANSITION_MAGIC);
1687 
1688         CRM_CHECK(a_magic != NULL && b_magic != NULL, sort_return(0, "No magic"));
1689         if (!decode_transition_magic(a_magic, &a_uuid, &a_id, NULL, NULL, NULL,
1690                                      NULL)) {
1691             sort_return(0, "bad magic a");
1692         }
1693         if (!decode_transition_magic(b_magic, &b_uuid, &b_id, NULL, NULL, NULL,
1694                                      NULL)) {
1695             sort_return(0, "bad magic b");
1696         }
1697         /* try to determine the relative age of the operation...
1698          * some pending operations (e.g. a start) may have been superseded
1699          *   by a subsequent stop
1700          *
1701          * [a|b]_id == -1 means it's a shutdown operation and _always_ comes last
1702          */
1703         if (!pcmk__str_eq(a_uuid, b_uuid, pcmk__str_casei) || a_id == b_id) {
1704             /*
1705              * some of the logic in here may be redundant...
1706              *
1707              * if the UUID from the TE doesn't match then one better
1708              *   be a pending operation.
1709              * pending operations don't survive between elections and joins
1710              *   because we query the LRM directly
1711              */
1712 
1713             if (b_call_id == -1) {
1714                 sort_return(-1, "transition + call");
1715 
1716             } else if (a_call_id == -1) {
1717                 sort_return(1, "transition + call");
1718             }
1719 
1720         } else if ((a_id >= 0 && a_id < b_id) || b_id == -1) {
1721             sort_return(-1, "transition");
1722 
1723         } else if ((b_id >= 0 && a_id > b_id) || a_id == -1) {
1724             sort_return(1, "transition");
1725         }
1726     }
1727 
1728     /* we should never end up here */
1729     CRM_CHECK(FALSE, sort_return(0, "default"));
1730 }
1731 
1732 gint
1733 sort_op_by_callid(gconstpointer a, gconstpointer b)
     /*  */
1734 {
1735     return pe__is_newer_op((const xmlNode *) a, (const xmlNode *) b);
1736 }
1737 
1738 /*!
1739  * \internal
1740  * \brief Create a new pseudo-action for a resource
1741  *
1742  * \param[in,out] rsc       Resource to create action for
1743  * \param[in]     task      Action name
1744  * \param[in]     optional  Whether action should be considered optional
1745  * \param[in]     runnable  Whethe action should be considered runnable
1746  *
1747  * \return New action object corresponding to arguments
1748  */
1749 pcmk_action_t *
1750 pe__new_rsc_pseudo_action(pcmk_resource_t *rsc, const char *task, bool optional,
     /*  */
1751                           bool runnable)
1752 {
1753     pcmk_action_t *action = NULL;
1754 
1755     pcmk__assert((rsc != NULL) && (task != NULL));
1756 
1757     action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0), task, NULL,
1758                            optional, rsc->priv->scheduler);
1759     pcmk__set_action_flags(action, pcmk__action_pseudo);
1760     if (runnable) {
1761         pcmk__set_action_flags(action, pcmk__action_runnable);
1762     }
1763     return action;
1764 }
1765 
1766 /*!
1767  * \internal
1768  * \brief Add the expected result to an action
1769  *
1770  * \param[in,out] action           Action to add expected result to
1771  * \param[in]     expected_result  Expected result to add
1772  *
1773  * \note This is more efficient than calling pcmk__insert_meta().
1774  */
1775 void
1776 pe__add_action_expected_result(pcmk_action_t *action, int expected_result)
     /*  */
1777 {
1778     pcmk__assert((action != NULL) && (action->meta != NULL));
1779 
1780     g_hash_table_insert(action->meta, pcmk__str_copy(PCMK__META_OP_TARGET_RC),
1781                         pcmk__itoa(expected_result));
1782 }
/* */
root/lib/pengine/pe_actions.c

DEFINITIONS
