root/lib/pacemaker/pcmk_scheduler.c

/* */

DEFINITIONS

1. check_params
2. failcount_clear_action_exists
3. check_failure_threshold
4. apply_exclusive_discovery
5. apply_stickiness
6. apply_shutdown_locks
7. apply_node_criteria
8. assign_resources
9. clear_failcounts_if_orphaned
10. schedule_resource_actions
11. is_managed
12. any_managed_resources
13. needs_fencing
14. needs_shutdown
15. add_nondc_fencing
16. schedule_fencing
17. schedule_fencing_and_shutdowns
18. log_resource_details
19. log_all_actions
20. log_unrunnable_actions
21. pcmk__schedule_actions
22. pcmk__init_scheduler

   1 /*
   2  * Copyright 2004-2025 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <crm/crm.h>
  13 #include <crm/cib.h>
  14 #include <crm/cib/internal.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/common/scheduler_internal.h>
  18 
  19 #include <glib.h>
  20 
  21 #include <crm/pengine/status.h>
  22 #include <pacemaker-internal.h>
  23 #include "libpacemaker_private.h"
  24 
  25 CRM_TRACE_INIT_DATA(pacemaker);
  26 
  27 /*!
  28  * \internal
  29  * \brief Do deferred action checks after assignment
  30  *
  31  * When unpacking the resource history, the scheduler checks for resource
  32  * configurations that have changed since an action was run. However, at that
  33  * time, bundles using the REMOTE_CONTAINER_HACK don't have their final
  34  * parameter information, so instead they add a deferred check to a list. This
  35  * function processes one entry in that list.
  36  *
  37  * \param[in,out] rsc     Resource that action history is for
  38  * \param[in,out] node    Node that action history is for
  39  * \param[in]     rsc_op  Action history entry
  40  * \param[in]     check   Type of deferred check to do
  41  */
  42 static void
  43 check_params(pcmk_resource_t *rsc, pcmk_node_t *node, const xmlNode *rsc_op,
     /*  */
  44              enum pcmk__check_parameters check)
  45 {
  46     const char *reason = NULL;
  47     pcmk__op_digest_t *digest_data = NULL;
  48 
  49     switch (check) {
  50         case pcmk__check_active:
  51             if (pcmk__check_action_config(rsc, node, rsc_op)
  52                 && pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
  53                                     NULL)) {
  54                 reason = "action definition changed";
  55             }
  56             break;
  57 
  58         case pcmk__check_last_failure:
  59             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
  60                                                 rsc->priv->scheduler);
  61             switch (digest_data->rc) {
  62                 case pcmk__digest_unknown:
  63                     crm_trace("Resource %s history entry %s on %s has "
  64                               "no digest to compare",
  65                               rsc->id, pcmk__xe_id(rsc_op), node->priv->id);
  66                     break;
  67                 case pcmk__digest_match:
  68                     break;
  69                 default:
  70                     reason = "resource parameters have changed";
  71                     break;
  72             }
  73             break;
  74     }
  75     if (reason != NULL) {
  76         pe__clear_failcount(rsc, node, reason, rsc->priv->scheduler);
  77     }
  78 }
  79 
  80 /*!
  81  * \internal
  82  * \brief Check whether a resource has failcount clearing scheduled on a node
  83  *
  84  * \param[in] node  Node to check
  85  * \param[in] rsc   Resource to check
  86  *
  87  * \return true if \p rsc has failcount clearing scheduled on \p node,
  88  *         otherwise false
  89  */
  90 static bool
  91 failcount_clear_action_exists(const pcmk_node_t *node,
     /*  */
  92                               const pcmk_resource_t *rsc)
  93 {
  94     GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
  95                                        TRUE);
  96 
  97     if (list != NULL) {
  98         g_list_free(list);
  99         return true;
 100     }
 101     return false;
 102 }
 103 
 104 /*!
 105  * \internal
 106  * \brief Ban a resource from a node if it reached its failure threshold there
 107  *
 108  * \param[in,out] data       Resource to check failure threshold for
 109  * \param[in]     user_data  Node to check resource on
 110  */
 111 static void
 112 check_failure_threshold(gpointer data, gpointer user_data)
     /*  */
 113 {
 114     pcmk_resource_t *rsc = data;
 115     const pcmk_node_t *node = user_data;
 116 
 117     // If this is a collective resource, apply recursively to children instead
 118     if (rsc->priv->children != NULL) {
 119         g_list_foreach(rsc->priv->children, check_failure_threshold,
 120                        user_data);
 121         return;
 122     }
 123 
 124     if (!failcount_clear_action_exists(node, rsc)) {
 125         /* Don't force the resource away from this node due to a failcount
 126          * that's going to be cleared.
 127          *
 128          * @TODO Failcount clearing can be scheduled in
 129          * pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
 130          * schedule_resource_actions() via check_params(). This runs well before
 131          * then, so it cannot detect those, meaning we might check the migration
 132          * threshold when we shouldn't. Worst case, we stop or move the
 133          * resource, then move it back in the next transition.
 134          */
 135         pcmk_resource_t *failed = NULL;
 136 
 137         if (pcmk__threshold_reached(rsc, node, &failed)) {
 138             resource_location(failed, node, -PCMK_SCORE_INFINITY,
 139                               "__fail_limit__", rsc->priv->scheduler);
 140         }
 141     }
 142 }
 143 
 144 /*!
 145  * \internal
 146  * \brief If resource has exclusive discovery, ban node if not allowed
 147  *
 148  * Location constraints have a PCMK_XA_RESOURCE_DISCOVERY option that allows
 149  * users to specify where probes are done for the affected resource. If this is
 150  * set to \c exclusive, probes will only be done on nodes listed in exclusive
 151  * constraints. This function bans the resource from the node if the node is not
 152  * listed.
 153  *
 154  * \param[in,out] data       Resource to check
 155  * \param[in]     user_data  Node to check resource on
 156  */
 157 static void
 158 apply_exclusive_discovery(gpointer data, gpointer user_data)
     /*  */
 159 {
 160     pcmk_resource_t *rsc = data;
 161     const pcmk_node_t *node = user_data;
 162 
 163     /* @TODO This checks rsc and the top rsc, but should probably check all
 164      * ancestors (a cloned group could have it set on the group)
 165      */
 166     if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
 167         || pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
 168                        pcmk__rsc_exclusive_probes)) {
 169         pcmk_node_t *match = NULL;
 170 
 171         // If this is a collective resource, apply recursively to children
 172         g_list_foreach(rsc->priv->children, apply_exclusive_discovery,
 173                        user_data);
 174 
 175         match = g_hash_table_lookup(rsc->priv->allowed_nodes,
 176                                     node->priv->id);
 177         if ((match != NULL)
 178             && (match->assign->probe_mode != pcmk__probe_exclusive)) {
 179             match->assign->score = -PCMK_SCORE_INFINITY;
 180         }
 181     }
 182 }
 183 
 184 /*!
 185  * \internal
 186  * \brief Apply stickiness to a resource if appropriate
 187  *
 188  * \param[in,out] data       Resource to check for stickiness
 189  * \param[in]     user_data  Ignored
 190  */
 191 static void
 192 apply_stickiness(gpointer data, gpointer user_data)
     /*  */
 193 {
 194     pcmk_resource_t *rsc = data;
 195     pcmk_node_t *node = NULL;
 196 
 197     // If this is a collective resource, apply recursively to children instead
 198     if (rsc->priv->children != NULL) {
 199         g_list_foreach(rsc->priv->children, apply_stickiness, NULL);
 200         return;
 201     }
 202 
 203     /* A resource is sticky if it is managed, has stickiness configured, and is
 204      * active on a single node.
 205      */
 206     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)
 207         || (rsc->priv->stickiness < 1)
 208         || !pcmk__list_of_1(rsc->priv->active_nodes)) {
 209         return;
 210     }
 211 
 212     node = rsc->priv->active_nodes->data;
 213 
 214     /* In a symmetric cluster, stickiness can always be used. In an
 215      * asymmetric cluster, we have to check whether the resource is still
 216      * allowed on the node, so we don't keep the resource somewhere it is no
 217      * longer explicitly enabled.
 218      */
 219     if (!pcmk_is_set(rsc->priv->scheduler->flags,
 220                      pcmk__sched_symmetric_cluster)
 221         && (g_hash_table_lookup(rsc->priv->allowed_nodes,
 222                                 node->priv->id) == NULL)) {
 223         pcmk__rsc_debug(rsc,
 224                         "Ignoring %s stickiness because the cluster is "
 225                         "asymmetric and %s is not explicitly allowed",
 226                         rsc->id, pcmk__node_name(node));
 227         return;
 228     }
 229 
 230     pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
 231                     rsc->id, rsc->priv->stickiness, pcmk__node_name(node));
 232     resource_location(rsc, node, rsc->priv->stickiness, "stickiness",
 233                       rsc->priv->scheduler);
 234 }
 235 
 236 /*!
 237  * \internal
 238  * \brief Apply shutdown locks for all resources as appropriate
 239  *
 240  * \param[in,out] scheduler  Scheduler data
 241  */
 242 static void
 243 apply_shutdown_locks(pcmk_scheduler_t *scheduler)
     /*  */
 244 {
 245     if (!pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
 246         return;
 247     }
 248     for (GList *iter = scheduler->priv->resources;
 249          iter != NULL; iter = iter->next) {
 250 
 251         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 252 
 253         rsc->priv->cmds->shutdown_lock(rsc);
 254     }
 255 }
 256 
 257 /*
 258  * \internal
 259  * \brief Apply node-specific scheduling criteria
 260  *
 261  * After the CIB has been unpacked, process node-specific scheduling criteria
 262  * including shutdown locks, location constraints, resource stickiness,
 263  * migration thresholds, and exclusive resource discovery.
 264  */
 265 static void
 266 apply_node_criteria(pcmk_scheduler_t *scheduler)
     /*  */
 267 {
 268     crm_trace("Applying node-specific scheduling criteria");
 269     apply_shutdown_locks(scheduler);
 270     pcmk__apply_locations(scheduler);
 271     g_list_foreach(scheduler->priv->resources, apply_stickiness, NULL);
 272 
 273     for (GList *node_iter = scheduler->nodes; node_iter != NULL;
 274          node_iter = node_iter->next) {
 275 
 276         for (GList *rsc_iter = scheduler->priv->resources;
 277              rsc_iter != NULL; rsc_iter = rsc_iter->next) {
 278 
 279             check_failure_threshold(rsc_iter->data, node_iter->data);
 280             apply_exclusive_discovery(rsc_iter->data, node_iter->data);
 281         }
 282     }
 283 }
 284 
 285 /*!
 286  * \internal
 287  * \brief Assign resources to nodes
 288  *
 289  * \param[in,out] scheduler  Scheduler data
 290  */
 291 static void
 292 assign_resources(pcmk_scheduler_t *scheduler)
     /*  */
 293 {
 294     GList *iter = NULL;
 295 
 296     crm_trace("Assigning resources to nodes");
 297 
 298     if (!pcmk__str_eq(scheduler->priv->placement_strategy, PCMK_VALUE_DEFAULT,
 299                       pcmk__str_casei)) {
 300         pcmk__sort_resources(scheduler);
 301     }
 302     pcmk__show_node_capacities("Original", scheduler);
 303 
 304     if (pcmk_is_set(scheduler->flags, pcmk__sched_have_remote_nodes)) {
 305         /* Assign remote connection resources first (which will also assign any
 306          * colocation dependencies). If the connection is migrating, always
 307          * prefer the partial migration target.
 308          */
 309         for (iter = scheduler->priv->resources;
 310              iter != NULL; iter = iter->next) {
 311 
 312             pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 313             const pcmk_node_t *target = rsc->priv->partial_migration_target;
 314 
 315             if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
 316                 pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
 317                                 rsc->id);
 318                 rsc->priv->cmds->assign(rsc, target, true);
 319             }
 320         }
 321     }
 322 
 323     /* now do the rest of the resources */
 324     for (iter = scheduler->priv->resources; iter != NULL; iter = iter->next) {
 325         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 326 
 327         if (!pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
 328             pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
 329                             rsc->priv->xml->name, rsc->id);
 330             rsc->priv->cmds->assign(rsc, NULL, true);
 331         }
 332     }
 333 
 334     pcmk__show_node_capacities("Remaining", scheduler);
 335 }
 336 
 337 /*!
 338  * \internal
 339  * \brief Schedule fail count clearing on online nodes if resource is orphaned
 340  *
 341  * \param[in,out] data       Resource to check
 342  * \param[in]     user_data  Ignored
 343  */
 344 static void
 345 clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
     /*  */
 346 {
 347     pcmk_resource_t *rsc = data;
 348 
 349     if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
 350         return;
 351     }
 352     crm_trace("Clear fail counts for orphaned resource %s", rsc->id);
 353 
 354     /* There's no need to recurse into rsc->private->children because those
 355      * should just be unassigned clone instances.
 356      */
 357 
 358     for (GList *iter = rsc->priv->scheduler->nodes;
 359          iter != NULL; iter = iter->next) {
 360 
 361         pcmk_node_t *node = (pcmk_node_t *) iter->data;
 362         pcmk_action_t *clear_op = NULL;
 363 
 364         if (!node->details->online) {
 365             continue;
 366         }
 367         if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
 368             continue;
 369         }
 370 
 371         clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
 372                                        rsc->priv->scheduler);
 373 
 374         /* We can't use order_action_then_stop() here because its
 375          * pcmk__ar_guest_allowed breaks things
 376          */
 377         pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
 378                            NULL, pcmk__ar_ordered, rsc->priv->scheduler);
 379     }
 380 }
 381 
 382 /*!
 383  * \internal
 384  * \brief Schedule any resource actions needed
 385  *
 386  * \param[in,out] scheduler  Scheduler data
 387  */
 388 static void
 389 schedule_resource_actions(pcmk_scheduler_t *scheduler)
     /*  */
 390 {
 391     // Process deferred action checks
 392     pcmk__foreach_param_check(scheduler, check_params);
 393     pcmk__free_param_checks(scheduler);
 394 
 395     if (pcmk_is_set(scheduler->flags, pcmk__sched_probe_resources)) {
 396         crm_trace("Scheduling probes");
 397         pcmk__schedule_probes(scheduler);
 398     }
 399 
 400     if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_removed_resources)) {
 401         g_list_foreach(scheduler->priv->resources, clear_failcounts_if_orphaned,
 402                        NULL);
 403     }
 404 
 405     crm_trace("Scheduling resource actions");
 406     for (GList *iter = scheduler->priv->resources;
 407          iter != NULL; iter = iter->next) {
 408 
 409         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 410 
 411         rsc->priv->cmds->create_actions(rsc);
 412     }
 413 }
 414 
 415 /*!
 416  * \internal
 417  * \brief Check whether a resource or any of its descendants are managed
 418  *
 419  * \param[in] rsc  Resource to check
 420  *
 421  * \return true if resource or any descendant is managed, otherwise false
 422  */
 423 static bool
 424 is_managed(const pcmk_resource_t *rsc)
     /*  */
 425 {
 426     if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 427         return true;
 428     }
 429     for (GList *iter = rsc->priv->children;
 430          iter != NULL; iter = iter->next) {
 431 
 432         if (is_managed((pcmk_resource_t *) iter->data)) {
 433             return true;
 434         }
 435     }
 436     return false;
 437 }
 438 
 439 /*!
 440  * \internal
 441  * \brief Check whether any resources in the cluster are managed
 442  *
 443  * \param[in] scheduler  Scheduler data
 444  *
 445  * \return true if any resource is managed, otherwise false
 446  */
 447 static bool
 448 any_managed_resources(const pcmk_scheduler_t *scheduler)
     /*  */
 449 {
 450     for (const GList *iter = scheduler->priv->resources;
 451          iter != NULL; iter = iter->next) {
 452         if (is_managed((const pcmk_resource_t *) iter->data)) {
 453             return true;
 454         }
 455     }
 456     return false;
 457 }
 458 
 459 /*!
 460  * \internal
 461  * \brief Check whether a node requires fencing
 462  *
 463  * \param[in] node          Node to check
 464  * \param[in] have_managed  Whether any resource in cluster is managed
 465  *
 466  * \return true if \p node should be fenced, otherwise false
 467  */
 468 static bool
 469 needs_fencing(const pcmk_node_t *node, bool have_managed)
     /*  */
 470 {
 471     return have_managed && node->details->unclean
 472            && pe_can_fence(node->priv->scheduler, node);
 473 }
 474 
 475 /*!
 476  * \internal
 477  * \brief Check whether a node requires shutdown
 478  *
 479  * \param[in] node          Node to check
 480  *
 481  * \return true if \p node should be shut down, otherwise false
 482  */
 483 static bool
 484 needs_shutdown(const pcmk_node_t *node)
     /*  */
 485 {
 486     if (pcmk__is_pacemaker_remote_node(node)) {
 487        /* Do not send shutdown actions for Pacemaker Remote nodes.
 488         * @TODO We might come up with a good use for this in the future.
 489         */
 490         return false;
 491     }
 492     return node->details->online && node->details->shutdown;
 493 }
 494 
 495 /*!
 496  * \internal
 497  * \brief Track and order non-DC fencing
 498  *
 499  * \param[in,out] list       List of existing non-DC fencing actions
 500  * \param[in,out] action     Fencing action to prepend to \p list
 501  * \param[in]     scheduler  Scheduler data
 502  *
 503  * \return (Possibly new) head of \p list
 504  */
 505 static GList *
 506 add_nondc_fencing(GList *list, pcmk_action_t *action,
     /*  */
 507                   const pcmk_scheduler_t *scheduler)
 508 {
 509     if (!pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)
 510         && (list != NULL)) {
 511         /* Concurrent fencing is disabled, so order each non-DC
 512          * fencing in a chain. If there is any DC fencing or
 513          * shutdown, it will be ordered after the last action in the
 514          * chain later.
 515          */
 516         order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
 517     }
 518     return g_list_prepend(list, action);
 519 }
 520 
 521 /*!
 522  * \internal
 523  * \brief Schedule a node for fencing
 524  *
 525  * \param[in,out] node      Node that requires fencing
 526  */
 527 static pcmk_action_t *
 528 schedule_fencing(pcmk_node_t *node)
     /*  */
 529 {
 530     pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
 531                                          FALSE, node->priv->scheduler);
 532 
 533     pcmk__sched_warn(node->priv->scheduler, "Scheduling node %s for fencing",
 534                      pcmk__node_name(node));
 535     pcmk__order_vs_fence(fencing, node->priv->scheduler);
 536     return fencing;
 537 }
 538 
 539 /*!
 540  * \internal
 541  * \brief Create and order node fencing and shutdown actions
 542  *
 543  * \param[in,out] scheduler  Scheduler data
 544  */
 545 static void
 546 schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
     /*  */
 547 {
 548     pcmk_action_t *dc_down = NULL;
 549     bool integrity_lost = false;
 550     bool have_managed = any_managed_resources(scheduler);
 551     GList *fencing_ops = NULL;
 552     GList *shutdown_ops = NULL;
 553 
 554     crm_trace("Scheduling fencing and shutdowns as needed");
 555     if (!have_managed) {
 556         crm_notice("No fencing will be done until there are resources "
 557                    "to manage");
 558     }
 559 
 560     // Check each node for whether it needs fencing or shutdown
 561     for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
 562         pcmk_node_t *node = (pcmk_node_t *) iter->data;
 563         pcmk_action_t *fencing = NULL;
 564         const bool is_dc = pcmk__same_node(node, scheduler->dc_node);
 565 
 566         /* Guest nodes are "fenced" by recovering their container resource,
 567          * so handle them separately.
 568          */
 569         if (pcmk__is_guest_or_bundle_node(node)) {
 570             if (pcmk_is_set(node->priv->flags, pcmk__node_remote_reset)
 571                 && have_managed && pe_can_fence(scheduler, node)) {
 572                 pcmk__fence_guest(node);
 573             }
 574             continue;
 575         }
 576 
 577         if (needs_fencing(node, have_managed)) {
 578             fencing = schedule_fencing(node);
 579 
 580             // Track DC and non-DC fence actions separately
 581             if (is_dc) {
 582                 dc_down = fencing;
 583             } else {
 584                 fencing_ops = add_nondc_fencing(fencing_ops, fencing,
 585                                                 scheduler);
 586             }
 587 
 588         } else if (needs_shutdown(node)) {
 589             pcmk_action_t *down_op = pcmk__new_shutdown_action(node);
 590 
 591             // Track DC and non-DC shutdown actions separately
 592             if (is_dc) {
 593                 dc_down = down_op;
 594             } else {
 595                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
 596             }
 597         }
 598 
 599         if ((fencing == NULL) && node->details->unclean) {
 600             integrity_lost = true;
 601             pcmk__config_warn("Node %s is unclean but cannot be fenced",
 602                               pcmk__node_name(node));
 603         }
 604     }
 605 
 606     if (integrity_lost) {
 607         if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
 608             pcmk__config_warn("Resource functionality and data integrity "
 609                               "cannot be guaranteed (configure, enable, "
 610                               "and test fencing to correct this)");
 611 
 612         } else if (!pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
 613             crm_notice("Unclean nodes will not be fenced until quorum is "
 614                        "attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
 615                        PCMK_VALUE_IGNORE);
 616         }
 617     }
 618 
 619     if (dc_down != NULL) {
 620         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
 621          * DC elections. However, we don't want to order non-DC shutdowns before
 622          * a DC *fencing*, because even though we don't want a node that's
 623          * shutting down to become DC, the DC fencing could be ordered before a
 624          * clone stop that's also ordered before the shutdowns, thus leading to
 625          * a graph loop.
 626          */
 627         if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
 628                          pcmk__str_none)) {
 629             pcmk__order_after_each(dc_down, shutdown_ops);
 630         }
 631 
 632         // Order any non-DC fencing before any DC fencing or shutdown
 633 
 634         if (pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)) {
 635             /* With concurrent fencing, order each non-DC fencing action
 636              * separately before any DC fencing or shutdown.
 637              */
 638             pcmk__order_after_each(dc_down, fencing_ops);
 639         } else if (fencing_ops != NULL) {
 640             /* Without concurrent fencing, the non-DC fencing actions are
 641              * already ordered relative to each other, so we just need to order
 642              * the DC fencing after the last action in the chain (which is the
 643              * first item in the list).
 644              */
 645             order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
 646                           pcmk__ar_ordered);
 647         }
 648     }
 649     g_list_free(fencing_ops);
 650     g_list_free(shutdown_ops);
 651 }
 652 
 653 static void
 654 log_resource_details(pcmk_scheduler_t *scheduler)
     /*  */
 655 {
 656     pcmk__output_t *out = scheduler->priv->out;
 657     GList *all = NULL;
 658 
 659     /* Due to the `crm_mon --node=` feature, out->message() for all the
 660      * resource-related messages expects a list of nodes that we are allowed to
 661      * output information for. Here, we create a wildcard to match all nodes.
 662      */
 663     all = g_list_prepend(all, (gpointer) "*");
 664 
 665     for (GList *item = scheduler->priv->resources;
 666          item != NULL; item = item->next) {
 667 
 668         pcmk_resource_t *rsc = (pcmk_resource_t *) item->data;
 669 
 670         // Log all resources except inactive orphans
 671         if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
 672             || (rsc->priv->orig_role != pcmk_role_stopped)) {
 673             out->message(out, (const char *) rsc->priv->xml->name, 0UL,
 674                          rsc, all, all);
 675         }
 676     }
 677 
 678     g_list_free(all);
 679 }
 680 
 681 static void
 682 log_all_actions(pcmk_scheduler_t *scheduler)
     /*  */
 683 {
 684     /* This only ever outputs to the log, so ignore whatever output object was
 685      * previously set and just log instead.
 686      */
 687     pcmk__output_t *prev_out = scheduler->priv->out;
 688     pcmk__output_t *out = NULL;
 689 
 690     if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
 691         return;
 692     }
 693 
 694     pe__register_messages(out);
 695     pcmk__register_lib_messages(out);
 696     pcmk__output_set_log_level(out, LOG_NOTICE);
 697     scheduler->priv->out = out;
 698 
 699     out->begin_list(out, NULL, NULL, "Actions");
 700     pcmk__output_actions(scheduler);
 701     out->end_list(out);
 702     out->finish(out, CRM_EX_OK, true, NULL);
 703     pcmk__output_free(out);
 704 
 705     scheduler->priv->out = prev_out;
 706 }
 707 
 708 /*!
 709  * \internal
 710  * \brief Log all required but unrunnable actions at trace level
 711  *
 712  * \param[in] scheduler  Scheduler data
 713  */
 714 static void
 715 log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
     /*  */
 716 {
 717     const uint64_t flags = pcmk__action_optional
 718                            |pcmk__action_runnable
 719                            |pcmk__action_pseudo;
 720 
 721     crm_trace("Required but unrunnable actions:");
 722     for (const GList *iter = scheduler->priv->actions;
 723          iter != NULL; iter = iter->next) {
 724 
 725         const pcmk_action_t *action = (const pcmk_action_t *) iter->data;
 726 
 727         if (!pcmk_any_flags_set(action->flags, flags)) {
 728             pcmk__log_action("\t", action, true);
 729         }
 730     }
 731 }
 732 
 733 /*!
 734  * \internal
 735  * \brief Run the scheduler for a given CIB
 736  *
 737  * \param[in,out] scheduler  Scheduler data
 738  */
 739 void
 740 pcmk__schedule_actions(pcmk_scheduler_t *scheduler)
     /*  */
 741 {
 742     cluster_status(scheduler);
 743     pcmk__set_assignment_methods(scheduler);
 744     pcmk__apply_node_health(scheduler);
 745     pcmk__unpack_constraints(scheduler);
 746     if (pcmk_is_set(scheduler->flags, pcmk__sched_validate_only)) {
 747         return;
 748     }
 749 
 750     if (!pcmk_is_set(scheduler->flags, pcmk__sched_location_only)
 751         && pcmk__is_daemon) {
 752         log_resource_details(scheduler);
 753     }
 754 
 755     apply_node_criteria(scheduler);
 756 
 757     if (pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
 758         return;
 759     }
 760 
 761     pcmk__create_internal_constraints(scheduler);
 762     pcmk__handle_rsc_config_changes(scheduler);
 763     assign_resources(scheduler);
 764     schedule_resource_actions(scheduler);
 765 
 766     /* Remote ordering constraints need to happen prior to calculating fencing
 767      * because it is one more place we can mark nodes as needing fencing.
 768      */
 769     pcmk__order_remote_connection_actions(scheduler);
 770 
 771     schedule_fencing_and_shutdowns(scheduler);
 772     pcmk__apply_orderings(scheduler);
 773     log_all_actions(scheduler);
 774     pcmk__create_graph(scheduler);
 775 
 776     if (get_crm_log_level() == LOG_TRACE) {
 777         log_unrunnable_actions(scheduler);
 778     }
 779 }
 780 
 781 /*!
 782  * \internal
 783  * \brief Initialize scheduler data
 784  *
 785  * Make our own copies of the CIB XML and date/time object, if they're not
 786  * \c NULL. This way we don't have to take ownership of the objects passed via
 787  * the API.
 788  *
 789  * This function is most useful for public API functions that want the caller
 790  * to retain ownership of the CIB object
 791  *
 792  * \param[in,out] out        Output object
 793  * \param[in]     input      The CIB XML to check (if \c NULL, use current CIB)
 794  * \param[in]     date       Date and time to use in the scheduler (if \c NULL,
 795  *                           use current date and time).  This can be used for
 796  *                           checking whether a rule is in effect at a certa
 797  *                           date and time.
 798  * \param[out]    scheduler  Where to store initialized scheduler data
 799  *
 800  * \return Standard Pacemaker return code
 801  */
 802 int
 803 pcmk__init_scheduler(pcmk__output_t *out, xmlNodePtr input, const crm_time_t *date,
     /*  */
 804                      pcmk_scheduler_t **scheduler)
 805 {
 806     // Allows for cleaner syntax than dereferencing the scheduler argument
 807     pcmk_scheduler_t *new_scheduler = NULL;
 808 
 809     new_scheduler = pcmk_new_scheduler();
 810     if (new_scheduler == NULL) {
 811         return ENOMEM;
 812     }
 813 
 814     pcmk__set_scheduler_flags(new_scheduler, pcmk__sched_no_counts);
 815 
 816     // Populate the scheduler data
 817 
 818     // Make our own copy of the given input or fetch the CIB and use that
 819     if (input != NULL) {
 820         new_scheduler->input = pcmk__xml_copy(NULL, input);
 821         if (new_scheduler->input == NULL) {
 822             out->err(out, "Failed to copy input XML");
 823             pcmk_free_scheduler(new_scheduler);
 824             return ENOMEM;
 825         }
 826 
 827     } else {
 828         int rc = cib__signon_query(out, NULL, &(new_scheduler->input));
 829 
 830         if (rc != pcmk_rc_ok) {
 831             pcmk_free_scheduler(new_scheduler);
 832             return rc;
 833         }
 834     }
 835 
 836     // Make our own copy of the given crm_time_t object; otherwise
 837     // cluster_status() populates with the current time
 838     if (date != NULL) {
 839         // pcmk_copy_time() guarantees non-NULL
 840         new_scheduler->priv->now = pcmk_copy_time(date);
 841     }
 842 
 843     // Unpack everything
 844     cluster_status(new_scheduler);
 845     *scheduler = new_scheduler;
 846 
 847     return pcmk_rc_ok;
 848 }

/* */