root/lib/pacemaker/pcmk_scheduler.c

/* */

DEFINITIONS

1. check_params
2. failcount_clear_action_exists
3. check_failure_threshold
4. apply_exclusive_discovery
5. apply_stickiness
6. apply_shutdown_locks
7. apply_node_criteria
8. assign_resources
9. clear_failcounts_if_orphaned
10. schedule_resource_actions
11. is_managed
12. any_managed_resources
13. needs_fencing
14. needs_shutdown
15. add_nondc_fencing
16. schedule_fencing
17. schedule_fencing_and_shutdowns
18. log_resource_details
19. log_all_actions
20. log_unrunnable_actions
21. unpack_cib
22. pcmk__schedule_actions
23. pcmk__init_scheduler

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <crm/crm.h>
  13 #include <crm/cib.h>
  14 #include <crm/cib/internal.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/common/scheduler_internal.h>
  18 
  19 #include <glib.h>
  20 
  21 #include <crm/pengine/status.h>
  22 #include <pacemaker-internal.h>
  23 #include "libpacemaker_private.h"
  24 
  25 CRM_TRACE_INIT_DATA(pacemaker);
  26 
  27 /*!
  28  * \internal
  29  * \brief Do deferred action checks after assignment
  30  *
  31  * When unpacking the resource history, the scheduler checks for resource
  32  * configurations that have changed since an action was run. However, at that
  33  * time, bundles using the REMOTE_CONTAINER_HACK don't have their final
  34  * parameter information, so instead they add a deferred check to a list. This
  35  * function processes one entry in that list.
  36  *
  37  * \param[in,out] rsc     Resource that action history is for
  38  * \param[in,out] node    Node that action history is for
  39  * \param[in]     rsc_op  Action history entry
  40  * \param[in]     check   Type of deferred check to do
  41  */
  42 static void
  43 check_params(pcmk_resource_t *rsc, pcmk_node_t *node, const xmlNode *rsc_op,
     /*  */
  44              enum pcmk__check_parameters check)
  45 {
  46     const char *reason = NULL;
  47     pcmk__op_digest_t *digest_data = NULL;
  48 
  49     switch (check) {
  50         case pcmk__check_active:
  51             if (pcmk__check_action_config(rsc, node, rsc_op)
  52                 && pe_get_failcount(node, rsc, NULL, pcmk__fc_effective,
  53                                     NULL)) {
  54                 reason = "action definition changed";
  55             }
  56             break;
  57 
  58         case pcmk__check_last_failure:
  59             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node,
  60                                                 rsc->priv->scheduler);
  61             switch (digest_data->rc) {
  62                 case pcmk__digest_unknown:
  63                     crm_trace("Resource %s history entry %s on %s has "
  64                               "no digest to compare",
  65                               rsc->id, pcmk__xe_id(rsc_op), node->priv->id);
  66                     break;
  67                 case pcmk__digest_match:
  68                     break;
  69                 default:
  70                     reason = "resource parameters have changed";
  71                     break;
  72             }
  73             break;
  74     }
  75     if (reason != NULL) {
  76         pe__clear_failcount(rsc, node, reason, rsc->priv->scheduler);
  77     }
  78 }
  79 
  80 /*!
  81  * \internal
  82  * \brief Check whether a resource has failcount clearing scheduled on a node
  83  *
  84  * \param[in] node  Node to check
  85  * \param[in] rsc   Resource to check
  86  *
  87  * \return true if \p rsc has failcount clearing scheduled on \p node,
  88  *         otherwise false
  89  */
  90 static bool
  91 failcount_clear_action_exists(const pcmk_node_t *node,
     /*  */
  92                               const pcmk_resource_t *rsc)
  93 {
  94     GList *list = pe__resource_actions(rsc, node, PCMK_ACTION_CLEAR_FAILCOUNT,
  95                                        TRUE);
  96 
  97     if (list != NULL) {
  98         g_list_free(list);
  99         return true;
 100     }
 101     return false;
 102 }
 103 
 104 /*!
 105  * \internal
 106  * \brief Ban a resource from a node if it reached its failure threshold there
 107  *
 108  * \param[in,out] data       Resource to check failure threshold for
 109  * \param[in]     user_data  Node to check resource on
 110  */
 111 static void
 112 check_failure_threshold(gpointer data, gpointer user_data)
     /*  */
 113 {
 114     pcmk_resource_t *rsc = data;
 115     const pcmk_node_t *node = user_data;
 116 
 117     // If this is a collective resource, apply recursively to children instead
 118     if (rsc->priv->children != NULL) {
 119         g_list_foreach(rsc->priv->children, check_failure_threshold,
 120                        user_data);
 121         return;
 122     }
 123 
 124     if (!failcount_clear_action_exists(node, rsc)) {
 125         /* Don't force the resource away from this node due to a failcount
 126          * that's going to be cleared.
 127          *
 128          * @TODO Failcount clearing can be scheduled in
 129          * pcmk__handle_rsc_config_changes() via process_rsc_history(), or in
 130          * schedule_resource_actions() via check_params(). This runs well before
 131          * then, so it cannot detect those, meaning we might check the migration
 132          * threshold when we shouldn't. Worst case, we stop or move the
 133          * resource, then move it back in the next transition.
 134          */
 135         pcmk_resource_t *failed = NULL;
 136 
 137         if (pcmk__threshold_reached(rsc, node, &failed)) {
 138             resource_location(failed, node, -PCMK_SCORE_INFINITY,
 139                               "__fail_limit__", rsc->priv->scheduler);
 140         }
 141     }
 142 }
 143 
 144 /*!
 145  * \internal
 146  * \brief If resource has exclusive discovery, ban node if not allowed
 147  *
 148  * Location constraints have a PCMK_XA_RESOURCE_DISCOVERY option that allows
 149  * users to specify where probes are done for the affected resource. If this is
 150  * set to \c exclusive, probes will only be done on nodes listed in exclusive
 151  * constraints. This function bans the resource from the node if the node is not
 152  * listed.
 153  *
 154  * \param[in,out] data       Resource to check
 155  * \param[in]     user_data  Node to check resource on
 156  */
 157 static void
 158 apply_exclusive_discovery(gpointer data, gpointer user_data)
     /*  */
 159 {
 160     pcmk_resource_t *rsc = data;
 161     const pcmk_node_t *node = user_data;
 162 
 163     if (pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)
 164         || pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
 165                        pcmk__rsc_exclusive_probes)) {
 166         pcmk_node_t *match = NULL;
 167 
 168         // If this is a collective resource, apply recursively to children
 169         g_list_foreach(rsc->priv->children, apply_exclusive_discovery,
 170                        user_data);
 171 
 172         match = g_hash_table_lookup(rsc->priv->allowed_nodes,
 173                                     node->priv->id);
 174         if ((match != NULL)
 175             && (match->assign->probe_mode != pcmk__probe_exclusive)) {
 176             match->assign->score = -PCMK_SCORE_INFINITY;
 177         }
 178     }
 179 }
 180 
 181 /*!
 182  * \internal
 183  * \brief Apply stickiness to a resource if appropriate
 184  *
 185  * \param[in,out] data       Resource to check for stickiness
 186  * \param[in]     user_data  Ignored
 187  */
 188 static void
 189 apply_stickiness(gpointer data, gpointer user_data)
     /*  */
 190 {
 191     pcmk_resource_t *rsc = data;
 192     pcmk_node_t *node = NULL;
 193 
 194     // If this is a collective resource, apply recursively to children instead
 195     if (rsc->priv->children != NULL) {
 196         g_list_foreach(rsc->priv->children, apply_stickiness, NULL);
 197         return;
 198     }
 199 
 200     /* A resource is sticky if it is managed, has stickiness configured, and is
 201      * active on a single node.
 202      */
 203     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)
 204         || (rsc->priv->stickiness < 1)
 205         || !pcmk__list_of_1(rsc->priv->active_nodes)) {
 206         return;
 207     }
 208 
 209     node = rsc->priv->active_nodes->data;
 210 
 211     /* In a symmetric cluster, stickiness can always be used. In an
 212      * asymmetric cluster, we have to check whether the resource is still
 213      * allowed on the node, so we don't keep the resource somewhere it is no
 214      * longer explicitly enabled.
 215      */
 216     if (!pcmk_is_set(rsc->priv->scheduler->flags,
 217                      pcmk__sched_symmetric_cluster)
 218         && (g_hash_table_lookup(rsc->priv->allowed_nodes,
 219                                 node->priv->id) == NULL)) {
 220         pcmk__rsc_debug(rsc,
 221                         "Ignoring %s stickiness because the cluster is "
 222                         "asymmetric and %s is not explicitly allowed",
 223                         rsc->id, pcmk__node_name(node));
 224         return;
 225     }
 226 
 227     pcmk__rsc_debug(rsc, "Resource %s has %d stickiness on %s",
 228                     rsc->id, rsc->priv->stickiness, pcmk__node_name(node));
 229     resource_location(rsc, node, rsc->priv->stickiness, "stickiness",
 230                       rsc->priv->scheduler);
 231 }
 232 
 233 /*!
 234  * \internal
 235  * \brief Apply shutdown locks for all resources as appropriate
 236  *
 237  * \param[in,out] scheduler  Scheduler data
 238  */
 239 static void
 240 apply_shutdown_locks(pcmk_scheduler_t *scheduler)
     /*  */
 241 {
 242     if (!pcmk_is_set(scheduler->flags, pcmk__sched_shutdown_lock)) {
 243         return;
 244     }
 245     for (GList *iter = scheduler->priv->resources;
 246          iter != NULL; iter = iter->next) {
 247 
 248         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 249 
 250         rsc->priv->cmds->shutdown_lock(rsc);
 251     }
 252 }
 253 
 254 /*
 255  * \internal
 256  * \brief Apply node-specific scheduling criteria
 257  *
 258  * After the CIB has been unpacked, process node-specific scheduling criteria
 259  * including shutdown locks, location constraints, resource stickiness,
 260  * migration thresholds, and exclusive resource discovery.
 261  */
 262 static void
 263 apply_node_criteria(pcmk_scheduler_t *scheduler)
     /*  */
 264 {
 265     crm_trace("Applying node-specific scheduling criteria");
 266     apply_shutdown_locks(scheduler);
 267     pcmk__apply_locations(scheduler);
 268     g_list_foreach(scheduler->priv->resources, apply_stickiness, NULL);
 269 
 270     for (GList *node_iter = scheduler->nodes; node_iter != NULL;
 271          node_iter = node_iter->next) {
 272 
 273         for (GList *rsc_iter = scheduler->priv->resources;
 274              rsc_iter != NULL; rsc_iter = rsc_iter->next) {
 275 
 276             check_failure_threshold(rsc_iter->data, node_iter->data);
 277             apply_exclusive_discovery(rsc_iter->data, node_iter->data);
 278         }
 279     }
 280 }
 281 
 282 /*!
 283  * \internal
 284  * \brief Assign resources to nodes
 285  *
 286  * \param[in,out] scheduler  Scheduler data
 287  */
 288 static void
 289 assign_resources(pcmk_scheduler_t *scheduler)
     /*  */
 290 {
 291     GList *iter = NULL;
 292 
 293     crm_trace("Assigning resources to nodes");
 294 
 295     if (!pcmk__str_eq(scheduler->priv->placement_strategy, PCMK_VALUE_DEFAULT,
 296                       pcmk__str_casei)) {
 297         pcmk__sort_resources(scheduler);
 298     }
 299     pcmk__show_node_capacities("Original", scheduler);
 300 
 301     if (pcmk_is_set(scheduler->flags, pcmk__sched_have_remote_nodes)) {
 302         /* Assign remote connection resources first (which will also assign any
 303          * colocation dependencies). If the connection is migrating, always
 304          * prefer the partial migration target.
 305          */
 306         for (iter = scheduler->priv->resources;
 307              iter != NULL; iter = iter->next) {
 308 
 309             pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 310             const pcmk_node_t *target = rsc->priv->partial_migration_target;
 311 
 312             if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
 313                 pcmk__rsc_trace(rsc, "Assigning remote connection resource '%s'",
 314                                 rsc->id);
 315                 rsc->priv->cmds->assign(rsc, target, true);
 316             }
 317         }
 318     }
 319 
 320     /* now do the rest of the resources */
 321     for (iter = scheduler->priv->resources; iter != NULL; iter = iter->next) {
 322         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 323 
 324         if (!pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
 325             pcmk__rsc_trace(rsc, "Assigning %s resource '%s'",
 326                             rsc->priv->xml->name, rsc->id);
 327             rsc->priv->cmds->assign(rsc, NULL, true);
 328         }
 329     }
 330 
 331     pcmk__show_node_capacities("Remaining", scheduler);
 332 }
 333 
 334 /*!
 335  * \internal
 336  * \brief Schedule fail count clearing on online nodes if resource is orphaned
 337  *
 338  * \param[in,out] data       Resource to check
 339  * \param[in]     user_data  Ignored
 340  */
 341 static void
 342 clear_failcounts_if_orphaned(gpointer data, gpointer user_data)
     /*  */
 343 {
 344     pcmk_resource_t *rsc = data;
 345 
 346     if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
 347         return;
 348     }
 349     crm_trace("Clear fail counts for orphaned resource %s", rsc->id);
 350 
 351     /* There's no need to recurse into rsc->private->children because those
 352      * should just be unassigned clone instances.
 353      */
 354 
 355     for (GList *iter = rsc->priv->scheduler->nodes;
 356          iter != NULL; iter = iter->next) {
 357 
 358         pcmk_node_t *node = (pcmk_node_t *) iter->data;
 359         pcmk_action_t *clear_op = NULL;
 360 
 361         if (!node->details->online) {
 362             continue;
 363         }
 364         if (pe_get_failcount(node, rsc, NULL, pcmk__fc_effective, NULL) == 0) {
 365             continue;
 366         }
 367 
 368         clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
 369                                        rsc->priv->scheduler);
 370 
 371         /* We can't use order_action_then_stop() here because its
 372          * pcmk__ar_guest_allowed breaks things
 373          */
 374         pcmk__new_ordering(clear_op->rsc, NULL, clear_op, rsc, stop_key(rsc),
 375                            NULL, pcmk__ar_ordered, rsc->priv->scheduler);
 376     }
 377 }
 378 
 379 /*!
 380  * \internal
 381  * \brief Schedule any resource actions needed
 382  *
 383  * \param[in,out] scheduler  Scheduler data
 384  */
 385 static void
 386 schedule_resource_actions(pcmk_scheduler_t *scheduler)
     /*  */
 387 {
 388     // Process deferred action checks
 389     pe__foreach_param_check(scheduler, check_params);
 390     pe__free_param_checks(scheduler);
 391 
 392     if (pcmk_is_set(scheduler->flags, pcmk__sched_probe_resources)) {
 393         crm_trace("Scheduling probes");
 394         pcmk__schedule_probes(scheduler);
 395     }
 396 
 397     if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_removed_resources)) {
 398         g_list_foreach(scheduler->priv->resources, clear_failcounts_if_orphaned,
 399                        NULL);
 400     }
 401 
 402     crm_trace("Scheduling resource actions");
 403     for (GList *iter = scheduler->priv->resources;
 404          iter != NULL; iter = iter->next) {
 405 
 406         pcmk_resource_t *rsc = (pcmk_resource_t *) iter->data;
 407 
 408         rsc->priv->cmds->create_actions(rsc);
 409     }
 410 }
 411 
 412 /*!
 413  * \internal
 414  * \brief Check whether a resource or any of its descendants are managed
 415  *
 416  * \param[in] rsc  Resource to check
 417  *
 418  * \return true if resource or any descendant is managed, otherwise false
 419  */
 420 static bool
 421 is_managed(const pcmk_resource_t *rsc)
     /*  */
 422 {
 423     if (pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 424         return true;
 425     }
 426     for (GList *iter = rsc->priv->children;
 427          iter != NULL; iter = iter->next) {
 428 
 429         if (is_managed((pcmk_resource_t *) iter->data)) {
 430             return true;
 431         }
 432     }
 433     return false;
 434 }
 435 
 436 /*!
 437  * \internal
 438  * \brief Check whether any resources in the cluster are managed
 439  *
 440  * \param[in] scheduler  Scheduler data
 441  *
 442  * \return true if any resource is managed, otherwise false
 443  */
 444 static bool
 445 any_managed_resources(const pcmk_scheduler_t *scheduler)
     /*  */
 446 {
 447     for (const GList *iter = scheduler->priv->resources;
 448          iter != NULL; iter = iter->next) {
 449         if (is_managed((const pcmk_resource_t *) iter->data)) {
 450             return true;
 451         }
 452     }
 453     return false;
 454 }
 455 
 456 /*!
 457  * \internal
 458  * \brief Check whether a node requires fencing
 459  *
 460  * \param[in] node          Node to check
 461  * \param[in] have_managed  Whether any resource in cluster is managed
 462  *
 463  * \return true if \p node should be fenced, otherwise false
 464  */
 465 static bool
 466 needs_fencing(const pcmk_node_t *node, bool have_managed)
     /*  */
 467 {
 468     return have_managed && node->details->unclean
 469            && pe_can_fence(node->priv->scheduler, node);
 470 }
 471 
 472 /*!
 473  * \internal
 474  * \brief Check whether a node requires shutdown
 475  *
 476  * \param[in] node          Node to check
 477  *
 478  * \return true if \p node should be shut down, otherwise false
 479  */
 480 static bool
 481 needs_shutdown(const pcmk_node_t *node)
     /*  */
 482 {
 483     if (pcmk__is_pacemaker_remote_node(node)) {
 484        /* Do not send shutdown actions for Pacemaker Remote nodes.
 485         * @TODO We might come up with a good use for this in the future.
 486         */
 487         return false;
 488     }
 489     return node->details->online && node->details->shutdown;
 490 }
 491 
 492 /*!
 493  * \internal
 494  * \brief Track and order non-DC fencing
 495  *
 496  * \param[in,out] list       List of existing non-DC fencing actions
 497  * \param[in,out] action     Fencing action to prepend to \p list
 498  * \param[in]     scheduler  Scheduler data
 499  *
 500  * \return (Possibly new) head of \p list
 501  */
 502 static GList *
 503 add_nondc_fencing(GList *list, pcmk_action_t *action,
     /*  */
 504                   const pcmk_scheduler_t *scheduler)
 505 {
 506     if (!pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)
 507         && (list != NULL)) {
 508         /* Concurrent fencing is disabled, so order each non-DC
 509          * fencing in a chain. If there is any DC fencing or
 510          * shutdown, it will be ordered after the last action in the
 511          * chain later.
 512          */
 513         order_actions((pcmk_action_t *) list->data, action, pcmk__ar_ordered);
 514     }
 515     return g_list_prepend(list, action);
 516 }
 517 
 518 /*!
 519  * \internal
 520  * \brief Schedule a node for fencing
 521  *
 522  * \param[in,out] node      Node that requires fencing
 523  */
 524 static pcmk_action_t *
 525 schedule_fencing(pcmk_node_t *node)
     /*  */
 526 {
 527     pcmk_action_t *fencing = pe_fence_op(node, NULL, FALSE, "node is unclean",
 528                                          FALSE, node->priv->scheduler);
 529 
 530     pcmk__sched_warn(node->priv->scheduler, "Scheduling node %s for fencing",
 531                      pcmk__node_name(node));
 532     pcmk__order_vs_fence(fencing, node->priv->scheduler);
 533     return fencing;
 534 }
 535 
 536 /*!
 537  * \internal
 538  * \brief Create and order node fencing and shutdown actions
 539  *
 540  * \param[in,out] scheduler  Scheduler data
 541  */
 542 static void
 543 schedule_fencing_and_shutdowns(pcmk_scheduler_t *scheduler)
     /*  */
 544 {
 545     pcmk_action_t *dc_down = NULL;
 546     bool integrity_lost = false;
 547     bool have_managed = any_managed_resources(scheduler);
 548     GList *fencing_ops = NULL;
 549     GList *shutdown_ops = NULL;
 550 
 551     crm_trace("Scheduling fencing and shutdowns as needed");
 552     if (!have_managed) {
 553         crm_notice("No fencing will be done until there are resources "
 554                    "to manage");
 555     }
 556 
 557     // Check each node for whether it needs fencing or shutdown
 558     for (GList *iter = scheduler->nodes; iter != NULL; iter = iter->next) {
 559         pcmk_node_t *node = (pcmk_node_t *) iter->data;
 560         pcmk_action_t *fencing = NULL;
 561         const bool is_dc = pcmk__same_node(node, scheduler->dc_node);
 562 
 563         /* Guest nodes are "fenced" by recovering their container resource,
 564          * so handle them separately.
 565          */
 566         if (pcmk__is_guest_or_bundle_node(node)) {
 567             if (pcmk_is_set(node->priv->flags, pcmk__node_remote_reset)
 568                 && have_managed && pe_can_fence(scheduler, node)) {
 569                 pcmk__fence_guest(node);
 570             }
 571             continue;
 572         }
 573 
 574         if (needs_fencing(node, have_managed)) {
 575             fencing = schedule_fencing(node);
 576 
 577             // Track DC and non-DC fence actions separately
 578             if (is_dc) {
 579                 dc_down = fencing;
 580             } else {
 581                 fencing_ops = add_nondc_fencing(fencing_ops, fencing,
 582                                                 scheduler);
 583             }
 584 
 585         } else if (needs_shutdown(node)) {
 586             pcmk_action_t *down_op = pcmk__new_shutdown_action(node);
 587 
 588             // Track DC and non-DC shutdown actions separately
 589             if (is_dc) {
 590                 dc_down = down_op;
 591             } else {
 592                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
 593             }
 594         }
 595 
 596         if ((fencing == NULL) && node->details->unclean) {
 597             integrity_lost = true;
 598             pcmk__config_warn("Node %s is unclean but cannot be fenced",
 599                               pcmk__node_name(node));
 600         }
 601     }
 602 
 603     if (integrity_lost) {
 604         if (!pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)) {
 605             pcmk__config_warn("Resource functionality and data integrity "
 606                               "cannot be guaranteed (configure, enable, "
 607                               "and test fencing to correct this)");
 608 
 609         } else if (!pcmk_is_set(scheduler->flags, pcmk__sched_quorate)) {
 610             crm_notice("Unclean nodes will not be fenced until quorum is "
 611                        "attained or " PCMK_OPT_NO_QUORUM_POLICY " is set to "
 612                        PCMK_VALUE_IGNORE);
 613         }
 614     }
 615 
 616     if (dc_down != NULL) {
 617         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
 618          * DC elections. However, we don't want to order non-DC shutdowns before
 619          * a DC *fencing*, because even though we don't want a node that's
 620          * shutting down to become DC, the DC fencing could be ordered before a
 621          * clone stop that's also ordered before the shutdowns, thus leading to
 622          * a graph loop.
 623          */
 624         if (pcmk__str_eq(dc_down->task, PCMK_ACTION_DO_SHUTDOWN,
 625                          pcmk__str_none)) {
 626             pcmk__order_after_each(dc_down, shutdown_ops);
 627         }
 628 
 629         // Order any non-DC fencing before any DC fencing or shutdown
 630 
 631         if (pcmk_is_set(scheduler->flags, pcmk__sched_concurrent_fencing)) {
 632             /* With concurrent fencing, order each non-DC fencing action
 633              * separately before any DC fencing or shutdown.
 634              */
 635             pcmk__order_after_each(dc_down, fencing_ops);
 636         } else if (fencing_ops != NULL) {
 637             /* Without concurrent fencing, the non-DC fencing actions are
 638              * already ordered relative to each other, so we just need to order
 639              * the DC fencing after the last action in the chain (which is the
 640              * first item in the list).
 641              */
 642             order_actions((pcmk_action_t *) fencing_ops->data, dc_down,
 643                           pcmk__ar_ordered);
 644         }
 645     }
 646     g_list_free(fencing_ops);
 647     g_list_free(shutdown_ops);
 648 }
 649 
 650 static void
 651 log_resource_details(pcmk_scheduler_t *scheduler)
     /*  */
 652 {
 653     pcmk__output_t *out = scheduler->priv->out;
 654     GList *all = NULL;
 655 
 656     /* Due to the `crm_mon --node=` feature, out->message() for all the
 657      * resource-related messages expects a list of nodes that we are allowed to
 658      * output information for. Here, we create a wildcard to match all nodes.
 659      */
 660     all = g_list_prepend(all, (gpointer) "*");
 661 
 662     for (GList *item = scheduler->priv->resources;
 663          item != NULL; item = item->next) {
 664 
 665         pcmk_resource_t *rsc = (pcmk_resource_t *) item->data;
 666 
 667         // Log all resources except inactive orphans
 668         if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)
 669             || (rsc->priv->orig_role != pcmk_role_stopped)) {
 670             out->message(out, (const char *) rsc->priv->xml->name, 0UL,
 671                          rsc, all, all);
 672         }
 673     }
 674 
 675     g_list_free(all);
 676 }
 677 
 678 static void
 679 log_all_actions(pcmk_scheduler_t *scheduler)
     /*  */
 680 {
 681     /* This only ever outputs to the log, so ignore whatever output object was
 682      * previously set and just log instead.
 683      */
 684     pcmk__output_t *prev_out = scheduler->priv->out;
 685     pcmk__output_t *out = NULL;
 686 
 687     if (pcmk__log_output_new(&out) != pcmk_rc_ok) {
 688         return;
 689     }
 690 
 691     pe__register_messages(out);
 692     pcmk__register_lib_messages(out);
 693     pcmk__output_set_log_level(out, LOG_NOTICE);
 694     scheduler->priv->out = out;
 695 
 696     out->begin_list(out, NULL, NULL, "Actions");
 697     pcmk__output_actions(scheduler);
 698     out->end_list(out);
 699     out->finish(out, CRM_EX_OK, true, NULL);
 700     pcmk__output_free(out);
 701 
 702     scheduler->priv->out = prev_out;
 703 }
 704 
 705 /*!
 706  * \internal
 707  * \brief Log all required but unrunnable actions at trace level
 708  *
 709  * \param[in] scheduler  Scheduler data
 710  */
 711 static void
 712 log_unrunnable_actions(const pcmk_scheduler_t *scheduler)
     /*  */
 713 {
 714     const uint64_t flags = pcmk__action_optional
 715                            |pcmk__action_runnable
 716                            |pcmk__action_pseudo;
 717 
 718     crm_trace("Required but unrunnable actions:");
 719     for (const GList *iter = scheduler->priv->actions;
 720          iter != NULL; iter = iter->next) {
 721 
 722         const pcmk_action_t *action = (const pcmk_action_t *) iter->data;
 723 
 724         if (!pcmk_any_flags_set(action->flags, flags)) {
 725             pcmk__log_action("\t", action, true);
 726         }
 727     }
 728 }
 729 
 730 /*!
 731  * \internal
 732  * \brief Unpack the CIB for scheduling
 733  *
 734  * \param[in,out] cib        CIB XML to unpack (may be NULL if already unpacked)
 735  * \param[in]     flags      Scheduler flags to set in addition to defaults
 736  * \param[in,out] scheduler  Scheduler data
 737  */
 738 static void
 739 unpack_cib(xmlNode *cib, unsigned long long flags, pcmk_scheduler_t *scheduler)
     /*  */
 740 {
 741     if (pcmk_is_set(scheduler->flags, pcmk__sched_have_status)) {
 742         crm_trace("Reusing previously calculated cluster status");
 743         pcmk__set_scheduler_flags(scheduler, flags);
 744         return;
 745     }
 746 
 747     pcmk__assert(cib != NULL);
 748     crm_trace("Calculating cluster status");
 749 
 750     /* This will zero the entire struct without freeing anything first, so
 751      * callers should never call pcmk__schedule_actions() with a populated data
 752      * set unless pcmk__sched_have_status is set (i.e. cluster_status() was
 753      * previously called, whether directly or via pcmk__schedule_actions()).
 754      */
 755     set_working_set_defaults(scheduler);
 756 
 757     pcmk__set_scheduler_flags(scheduler, flags);
 758     scheduler->input = cib;
 759     cluster_status(scheduler); // Sets pcmk__sched_have_status
 760 }
 761 
 762 /*!
 763  * \internal
 764  * \brief Run the scheduler for a given CIB
 765  *
 766  * \param[in,out] cib        CIB XML to use as scheduler input
 767  * \param[in]     flags      Scheduler flags to set in addition to defaults
 768  * \param[in,out] scheduler  Scheduler data
 769  */
 770 void
 771 pcmk__schedule_actions(xmlNode *cib, unsigned long long flags,
     /*  */
 772                        pcmk_scheduler_t *scheduler)
 773 {
 774     unpack_cib(cib, flags, scheduler);
 775     pcmk__set_assignment_methods(scheduler);
 776     pcmk__apply_node_health(scheduler);
 777     pcmk__unpack_constraints(scheduler);
 778     if (pcmk_is_set(scheduler->flags, pcmk__sched_validate_only)) {
 779         return;
 780     }
 781 
 782     if (!pcmk_is_set(scheduler->flags, pcmk__sched_location_only)
 783         && pcmk__is_daemon) {
 784         log_resource_details(scheduler);
 785     }
 786 
 787     apply_node_criteria(scheduler);
 788 
 789     if (pcmk_is_set(scheduler->flags, pcmk__sched_location_only)) {
 790         return;
 791     }
 792 
 793     pcmk__create_internal_constraints(scheduler);
 794     pcmk__handle_rsc_config_changes(scheduler);
 795     assign_resources(scheduler);
 796     schedule_resource_actions(scheduler);
 797 
 798     /* Remote ordering constraints need to happen prior to calculating fencing
 799      * because it is one more place we can mark nodes as needing fencing.
 800      */
 801     pcmk__order_remote_connection_actions(scheduler);
 802 
 803     schedule_fencing_and_shutdowns(scheduler);
 804     pcmk__apply_orderings(scheduler);
 805     log_all_actions(scheduler);
 806     pcmk__create_graph(scheduler);
 807 
 808     if (get_crm_log_level() == LOG_TRACE) {
 809         log_unrunnable_actions(scheduler);
 810     }
 811 }
 812 
 813 /*!
 814  * \internal
 815  * \brief Initialize scheduler data
 816  *
 817  * Make our own copies of the CIB XML and date/time object, if they're not
 818  * \c NULL. This way we don't have to take ownership of the objects passed via
 819  * the API.
 820  *
 821  * This function is most useful for public API functions that want the caller
 822  * to retain ownership of the CIB object
 823  *
 824  * \param[in,out] out        Output object
 825  * \param[in]     input      The CIB XML to check (if \c NULL, use current CIB)
 826  * \param[in]     date       Date and time to use in the scheduler (if \c NULL,
 827  *                           use current date and time).  This can be used for
 828  *                           checking whether a rule is in effect at a certa
 829  *                           date and time.
 830  * \param[out]    scheduler  Where to store initialized scheduler data
 831  *
 832  * \return Standard Pacemaker return code
 833  */
 834 int
 835 pcmk__init_scheduler(pcmk__output_t *out, xmlNodePtr input, const crm_time_t *date,
     /*  */
 836                      pcmk_scheduler_t **scheduler)
 837 {
 838     // Allows for cleaner syntax than dereferencing the scheduler argument
 839     pcmk_scheduler_t *new_scheduler = NULL;
 840 
 841     new_scheduler = pe_new_working_set();
 842     if (new_scheduler == NULL) {
 843         return ENOMEM;
 844     }
 845 
 846     pcmk__set_scheduler_flags(new_scheduler, pcmk__sched_no_counts);
 847 
 848     // Populate the scheduler data
 849 
 850     // Make our own copy of the given input or fetch the CIB and use that
 851     if (input != NULL) {
 852         new_scheduler->input = pcmk__xml_copy(NULL, input);
 853         if (new_scheduler->input == NULL) {
 854             out->err(out, "Failed to copy input XML");
 855             pe_free_working_set(new_scheduler);
 856             return ENOMEM;
 857         }
 858 
 859     } else {
 860         int rc = cib__signon_query(out, NULL, &(new_scheduler->input));
 861 
 862         if (rc != pcmk_rc_ok) {
 863             pe_free_working_set(new_scheduler);
 864             return rc;
 865         }
 866     }
 867 
 868     // Make our own copy of the given crm_time_t object; otherwise
 869     // cluster_status() populates with the current time
 870     if (date != NULL) {
 871         // pcmk_copy_time() guarantees non-NULL
 872         new_scheduler->priv->now = pcmk_copy_time(date);
 873     }
 874 
 875     // Unpack everything
 876     cluster_status(new_scheduler);
 877     *scheduler = new_scheduler;
 878 
 879     return pcmk_rc_ok;
 880 }

/* */