lib/pacemaker/pcmk_sched

/* */
This source file includes following definitions.
sorted_allowed_nodes
assign_best_node
apply_this_with
remote_connection_assigned
pcmk__primitive_assign
schedule_restart_actions
set_default_next_role
create_pending_start
schedule_role_transition_actions
pcmk__primitive_create_actions
rsc_avoids_remote_nodes
allowed_nodes_as_list
pcmk__primitive_internal_constraints
pcmk__primitive_apply_coloc_score
pcmk__with_primitive_colocations
pcmk__primitive_with_colocations
pcmk__primitive_action_flags
is_expected_node
stop_resource
start_resource
promote_resource
demote_resource
assert_role_error
pcmk__schedule_cleanup
pcmk__primitive_add_graph_meta
pcmk__primitive_add_utilization
shutdown_time
ban_if_not_locked
pcmk__primitive_shutdown_lock
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdbool.h>
  13 #include <stdint.h>                 // uint8_t, uint32_t
  14 
  15 #include <crm/common/xml.h>
  16 #include <pacemaker-internal.h>
  17 
  18 #include "libpacemaker_private.h"
  19 
  20 static void stop_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
  21                           bool optional);
  22 static void start_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
  23                            bool optional);
  24 static void demote_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
  25                             bool optional);
  26 static void promote_resource(pcmk_resource_t *rsc, pcmk_node_t *node,
  27                              bool optional);
  28 static void assert_role_error(pcmk_resource_t *rsc, pcmk_node_t *node,
  29                               bool optional);
  30 
  31 #define RSC_ROLE_MAX    (pcmk_role_promoted + 1)
  32 
  33 static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
  34     /* This array lists the immediate next role when transitioning from one role
  35      * to a target role. For example, when going from Stopped to Promoted, the
  36      * next role is Unpromoted, because the resource must be started before it
  37      * can be promoted. The current state then becomes Started, which is fed
  38      * into this array again, giving a next role of Promoted.
  39      *
  40      * Current role       Immediate next role   Final target role
  41      * ------------       -------------------   -----------------
  42      */
  43     /* Unknown */       { pcmk_role_unknown,    /* Unknown */
  44                           pcmk_role_stopped,    /* Stopped */
  45                           pcmk_role_stopped,    /* Started */
  46                           pcmk_role_stopped,    /* Unpromoted */
  47                           pcmk_role_stopped,    /* Promoted */
  48                         },
  49     /* Stopped */       { pcmk_role_stopped,    /* Unknown */
  50                           pcmk_role_stopped,    /* Stopped */
  51                           pcmk_role_started,    /* Started */
  52                           pcmk_role_unpromoted, /* Unpromoted */
  53                           pcmk_role_unpromoted, /* Promoted */
  54                         },
  55     /* Started */       { pcmk_role_stopped,    /* Unknown */
  56                           pcmk_role_stopped,    /* Stopped */
  57                           pcmk_role_started,    /* Started */
  58                           pcmk_role_unpromoted, /* Unpromoted */
  59                           pcmk_role_promoted,   /* Promoted */
  60                         },
  61     /* Unpromoted */    { pcmk_role_stopped,    /* Unknown */
  62                           pcmk_role_stopped,    /* Stopped */
  63                           pcmk_role_stopped,    /* Started */
  64                           pcmk_role_unpromoted, /* Unpromoted */
  65                           pcmk_role_promoted,   /* Promoted */
  66                         },
  67     /* Promoted  */     { pcmk_role_stopped,    /* Unknown */
  68                           pcmk_role_unpromoted, /* Stopped */
  69                           pcmk_role_unpromoted, /* Started */
  70                           pcmk_role_unpromoted, /* Unpromoted */
  71                           pcmk_role_promoted,   /* Promoted */
  72                         },
  73 };
  74 
  75 /*!
  76  * \internal
  77  * \brief Function to schedule actions needed for a role change
  78  *
  79  * \param[in,out] rsc       Resource whose role is changing
  80  * \param[in,out] node      Node where resource will be in its next role
  81  * \param[in]     optional  Whether scheduled actions should be optional
  82  */
  83 typedef void (*rsc_transition_fn)(pcmk_resource_t *rsc, pcmk_node_t *node,
  84                                   bool optional);
  85 
  86 static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
  87     /* This array lists the function needed to transition directly from one role
  88      * to another. NULL indicates that nothing is needed.
  89      *
  90      * Current role         Transition function             Next role
  91      * ------------         -------------------             ----------
  92      */
  93     /* Unknown */       {   assert_role_error,              /* Unknown */
  94                             stop_resource,                  /* Stopped */
  95                             assert_role_error,              /* Started */
  96                             assert_role_error,              /* Unpromoted */
  97                             assert_role_error,              /* Promoted */
  98                         },
  99     /* Stopped */       {   assert_role_error,              /* Unknown */
 100                             NULL,                           /* Stopped */
 101                             start_resource,                 /* Started */
 102                             start_resource,                 /* Unpromoted */
 103                             assert_role_error,              /* Promoted */
 104                         },
 105     /* Started */       {   assert_role_error,              /* Unknown */
 106                             stop_resource,                  /* Stopped */
 107                             NULL,                           /* Started */
 108                             NULL,                           /* Unpromoted */
 109                             promote_resource,               /* Promoted */
 110                         },
 111     /* Unpromoted */    {   assert_role_error,              /* Unknown */
 112                             stop_resource,                  /* Stopped */
 113                             stop_resource,                  /* Started */
 114                             NULL,                           /* Unpromoted */
 115                             promote_resource,               /* Promoted */
 116                         },
 117     /* Promoted  */     {   assert_role_error,              /* Unknown */
 118                             demote_resource,                /* Stopped */
 119                             demote_resource,                /* Started */
 120                             demote_resource,                /* Unpromoted */
 121                             NULL,                           /* Promoted */
 122                         },
 123 };
 124 
 125 /*!
 126  * \internal
 127  * \brief Get a list of a resource's allowed nodes sorted by node score
 128  *
 129  * \param[in] rsc  Resource to check
 130  *
 131  * \return List of allowed nodes sorted by node score
 132  */
 133 static GList *
 134 sorted_allowed_nodes(const pcmk_resource_t *rsc)
     /*  */
 135 {
 136     if (rsc->priv->allowed_nodes != NULL) {
 137         GList *nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
 138 
 139         if (nodes != NULL) {
 140             return pcmk__sort_nodes(nodes, pcmk__current_node(rsc));
 141         }
 142     }
 143     return NULL;
 144 }
 145 
 146 /*!
 147  * \internal
 148  * \brief Assign a resource to its best allowed node, if possible
 149  *
 150  * \param[in,out] rsc           Resource to choose a node for
 151  * \param[in]     prefer        If not \c NULL, prefer this node when all else
 152  *                              equal
 153  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
 154  *                              node, set next role to stopped and update
 155  *                              existing actions
 156  *
 157  * \return true if \p rsc could be assigned to a node, otherwise false
 158  *
 159  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
 160  *       completely undo the assignment. A successful assignment can be either
 161  *       undone or left alone as final. A failed assignment has the same effect
 162  *       as calling pcmk__unassign_resource(); there are no side effects on
 163  *       roles or actions.
 164  */
 165 static bool
 166 assign_best_node(pcmk_resource_t *rsc, const pcmk_node_t *prefer,
     /*  */
 167                  bool stop_if_fail)
 168 {
 169     GList *nodes = NULL;
 170     pcmk_node_t *chosen = NULL;
 171     pcmk_node_t *best = NULL;
 172     const pcmk_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);
 173 
 174     if (prefer == NULL) {
 175         prefer = most_free_node;
 176     }
 177 
 178     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
 179         // We've already finished assignment of resources to nodes
 180         return rsc->priv->assigned_node != NULL;
 181     }
 182 
 183     // Sort allowed nodes by score
 184     nodes = sorted_allowed_nodes(rsc);
 185     if (nodes != NULL) {
 186         best = (pcmk_node_t *) nodes->data; // First node has best score
 187     }
 188 
 189     if ((prefer != NULL) && (nodes != NULL)) {
 190         // Get the allowed node version of prefer
 191         chosen = g_hash_table_lookup(rsc->priv->allowed_nodes,
 192                                      prefer->priv->id);
 193 
 194         if (chosen == NULL) {
 195             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unknown",
 196                             pcmk__node_name(prefer), rsc->id);
 197 
 198         /* Favor the preferred node as long as its score is at least as good as
 199          * the best allowed node's.
 200          *
 201          * An alternative would be to favor the preferred node even if the best
 202          * node is better, when the best node's score is less than INFINITY.
 203          */
 204         } else if (chosen->assign->score < best->assign->score) {
 205             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
 206                             pcmk__node_name(chosen), rsc->id);
 207             chosen = NULL;
 208 
 209         } else if (!pcmk__node_available(chosen, true, false)) {
 210             pcmk__rsc_trace(rsc, "Preferred node %s for %s was unavailable",
 211                             pcmk__node_name(chosen), rsc->id);
 212             chosen = NULL;
 213 
 214         } else {
 215             pcmk__rsc_trace(rsc,
 216                             "Chose preferred node %s for %s "
 217                             "(ignoring %d candidates)",
 218                             pcmk__node_name(chosen), rsc->id,
 219                             g_list_length(nodes));
 220         }
 221     }
 222 
 223     if ((chosen == NULL) && (best != NULL)) {
 224         /* Either there is no preferred node, or the preferred node is not
 225          * suitable, but another node is allowed to run the resource.
 226          */
 227 
 228         chosen = best;
 229 
 230         if (!pcmk__is_unique_clone(rsc->priv->parent)
 231             && (chosen->assign->score > 0) // Zero not acceptable
 232             && pcmk__node_available(chosen, false, false)) {
 233             /* If the resource is already running on a node, prefer that node if
 234              * it is just as good as the chosen node.
 235              *
 236              * We don't do this for unique clone instances, because
 237              * pcmk__assign_instances() has already assigned instances to their
 238              * running nodes when appropriate, and if we get here, we don't want
 239              * remaining unassigned instances to prefer a node that's already
 240              * running another instance.
 241              */
 242             pcmk_node_t *running = pcmk__current_node(rsc);
 243 
 244             if (running == NULL) {
 245                 // Nothing to do
 246 
 247             } else if (!pcmk__node_available(running, true, false)) {
 248                 pcmk__rsc_trace(rsc,
 249                                 "Current node for %s (%s) can't run resources",
 250                                 rsc->id, pcmk__node_name(running));
 251 
 252             } else {
 253                 int nodes_with_best_score = 1;
 254 
 255                 for (GList *iter = nodes->next; iter; iter = iter->next) {
 256                     pcmk_node_t *allowed = (pcmk_node_t *) iter->data;
 257 
 258                     if (allowed->assign->score != chosen->assign->score) {
 259                         // The nodes are sorted by score, so no more are equal
 260                         break;
 261                     }
 262                     if (pcmk__same_node(allowed, running)) {
 263                         // Scores are equal, so prefer the current node
 264                         chosen = allowed;
 265                     }
 266                     nodes_with_best_score++;
 267                 }
 268 
 269                 if (nodes_with_best_score > 1) {
 270                     uint8_t log_level = LOG_INFO;
 271 
 272                     if (chosen->assign->score >= PCMK_SCORE_INFINITY) {
 273                         log_level = LOG_WARNING;
 274                     }
 275                     do_crm_log(log_level,
 276                                "Chose %s for %s from %d nodes with score %s",
 277                                pcmk__node_name(chosen), rsc->id,
 278                                nodes_with_best_score,
 279                                pcmk_readable_score(chosen->assign->score));
 280                 }
 281             }
 282         }
 283 
 284         pcmk__rsc_trace(rsc, "Chose %s for %s from %d candidates",
 285                         pcmk__node_name(chosen), rsc->id, g_list_length(nodes));
 286     }
 287 
 288     pcmk__assign_resource(rsc, chosen, false, stop_if_fail);
 289     g_list_free(nodes);
 290     return rsc->priv->assigned_node != NULL;
 291 }
 292 
 293 /*!
 294  * \internal
 295  * \brief Apply a "this with" colocation to a node's allowed node scores
 296  *
 297  * \param[in,out] colocation  Colocation to apply
 298  * \param[in,out] rsc         Resource being assigned
 299  */
 300 static void
 301 apply_this_with(pcmk__colocation_t *colocation, pcmk_resource_t *rsc)
     /*  */
 302 {
 303     GHashTable *archive = NULL;
 304     pcmk_resource_t *other = colocation->primary;
 305 
 306     // In certain cases, we will need to revert the node scores
 307     if ((colocation->dependent_role >= pcmk_role_promoted)
 308         || ((colocation->score < 0)
 309             && (colocation->score > -PCMK_SCORE_INFINITY))) {
 310         archive = pcmk__copy_node_table(rsc->priv->allowed_nodes);
 311     }
 312 
 313     if (pcmk_is_set(other->flags, pcmk__rsc_unassigned)) {
 314         pcmk__rsc_trace(rsc,
 315                         "%s: Assigning colocation %s primary %s first"
 316                         "(score=%d role=%s)",
 317                         rsc->id, colocation->id, other->id,
 318                         colocation->score,
 319                         pcmk_role_text(colocation->dependent_role));
 320         other->priv->cmds->assign(other, NULL, true);
 321     }
 322 
 323     // Apply the colocation score to this resource's allowed node scores
 324     rsc->priv->cmds->apply_coloc_score(rsc, other, colocation, true);
 325     if ((archive != NULL)
 326         && !pcmk__any_node_available(rsc->priv->allowed_nodes)) {
 327         pcmk__rsc_info(rsc,
 328                        "%s: Reverting scores from colocation with %s "
 329                        "because no nodes allowed",
 330                        rsc->id, other->id);
 331         g_hash_table_destroy(rsc->priv->allowed_nodes);
 332         rsc->priv->allowed_nodes = archive;
 333         archive = NULL;
 334     }
 335     if (archive != NULL) {
 336         g_hash_table_destroy(archive);
 337     }
 338 }
 339 
 340 /*!
 341  * \internal
 342  * \brief Update a Pacemaker Remote node once its connection has been assigned
 343  *
 344  * \param[in] connection  Connection resource that has been assigned
 345  */
 346 static void
 347 remote_connection_assigned(const pcmk_resource_t *connection)
     /*  */
 348 {
 349     pcmk_node_t *remote_node = pcmk_find_node(connection->priv->scheduler,
 350                                               connection->id);
 351 
 352     CRM_CHECK(remote_node != NULL, return);
 353 
 354     if ((connection->priv->assigned_node != NULL)
 355         && (connection->priv->next_role != pcmk_role_stopped)) {
 356 
 357         crm_trace("Pacemaker Remote node %s will be online",
 358                   remote_node->priv->id);
 359         remote_node->details->online = TRUE;
 360         if (!pcmk_is_set(remote_node->priv->flags, pcmk__node_seen)) {
 361             // Avoid unnecessary fence, since we will attempt connection
 362             remote_node->details->unclean = FALSE;
 363         }
 364 
 365     } else {
 366         crm_trace("Pacemaker Remote node %s will be shut down "
 367                   "(%sassigned connection's next role is %s)",
 368                   remote_node->priv->id,
 369                   ((connection->priv->assigned_node == NULL)? "un" : ""),
 370                   pcmk_role_text(connection->priv->next_role));
 371         remote_node->details->shutdown = TRUE;
 372     }
 373 }
 374 
 375 /*!
 376  * \internal
 377  * \brief Assign a primitive resource to a node
 378  *
 379  * \param[in,out] rsc           Resource to assign to a node
 380  * \param[in]     prefer        Node to prefer, if all else is equal
 381  * \param[in]     stop_if_fail  If \c true and \p rsc can't be assigned to a
 382  *                              node, set next role to stopped and update
 383  *                              existing actions
 384  *
 385  * \return Node that \p rsc is assigned to, if assigned entirely to one node
 386  *
 387  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
 388  *       completely undo the assignment. A successful assignment can be either
 389  *       undone or left alone as final. A failed assignment has the same effect
 390  *       as calling pcmk__unassign_resource(); there are no side effects on
 391  *       roles or actions.
 392  */
 393 pcmk_node_t *
 394 pcmk__primitive_assign(pcmk_resource_t *rsc, const pcmk_node_t *prefer,
     /*  */
 395                        bool stop_if_fail)
 396 {
 397     GList *this_with_colocations = NULL;
 398     GList *with_this_colocations = NULL;
 399     GList *iter = NULL;
 400     pcmk_resource_t *parent = NULL;
 401     pcmk__colocation_t *colocation = NULL;
 402     pcmk_scheduler_t *scheduler = NULL;
 403 
 404     pcmk__assert(pcmk__is_primitive(rsc));
 405     scheduler = rsc->priv->scheduler;
 406     parent = rsc->priv->parent;
 407 
 408     // Never assign a child without parent being assigned first
 409     if ((parent != NULL) && !pcmk_is_set(parent->flags, pcmk__rsc_assigning)) {
 410         pcmk__rsc_debug(rsc, "%s: Assigning parent %s first",
 411                         rsc->id, parent->id);
 412         parent->priv->cmds->assign(parent, prefer, stop_if_fail);
 413     }
 414 
 415     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
 416         // Assignment has already been done
 417         const char *node_name = "no node";
 418 
 419         if (rsc->priv->assigned_node != NULL) {
 420             node_name = pcmk__node_name(rsc->priv->assigned_node);
 421         }
 422         pcmk__rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id, node_name);
 423         return rsc->priv->assigned_node;
 424     }
 425 
 426     // Ensure we detect assignment loops
 427     if (pcmk_is_set(rsc->flags, pcmk__rsc_assigning)) {
 428         pcmk__rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
 429         return NULL;
 430     }
 431     pcmk__set_rsc_flags(rsc, pcmk__rsc_assigning);
 432 
 433     pe__show_node_scores(true, rsc, "Pre-assignment",
 434                          rsc->priv->allowed_nodes, scheduler);
 435 
 436     this_with_colocations = pcmk__this_with_colocations(rsc);
 437     with_this_colocations = pcmk__with_this_colocations(rsc);
 438 
 439     // Apply mandatory colocations first, to satisfy as many as possible
 440     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
 441         colocation = iter->data;
 442 
 443         if ((colocation->score <= -PCMK_SCORE_INFINITY)
 444             || (colocation->score >= PCMK_SCORE_INFINITY)) {
 445             apply_this_with(colocation, rsc);
 446         }
 447     }
 448     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
 449         colocation = iter->data;
 450 
 451         if ((colocation->score <= -PCMK_SCORE_INFINITY)
 452             || (colocation->score >= PCMK_SCORE_INFINITY)) {
 453             pcmk__add_dependent_scores(colocation, rsc);
 454         }
 455     }
 456 
 457     pe__show_node_scores(true, rsc, "Mandatory-colocations",
 458                          rsc->priv->allowed_nodes, scheduler);
 459 
 460     // Then apply optional colocations
 461     for (iter = this_with_colocations; iter != NULL; iter = iter->next) {
 462         colocation = iter->data;
 463 
 464         if ((colocation->score > -PCMK_SCORE_INFINITY)
 465             && (colocation->score < PCMK_SCORE_INFINITY)) {
 466             apply_this_with(colocation, rsc);
 467         }
 468     }
 469     for (iter = with_this_colocations; iter != NULL; iter = iter->next) {
 470         colocation = iter->data;
 471 
 472         if ((colocation->score > -PCMK_SCORE_INFINITY)
 473             && (colocation->score < PCMK_SCORE_INFINITY)) {
 474             pcmk__add_dependent_scores(colocation, rsc);
 475         }
 476     }
 477 
 478     g_list_free(this_with_colocations);
 479     g_list_free(with_this_colocations);
 480 
 481     if (rsc->priv->next_role == pcmk_role_stopped) {
 482         pcmk__rsc_trace(rsc,
 483                         "Banning %s from all nodes because it will be stopped",
 484                         rsc->id);
 485         resource_location(rsc, NULL, -PCMK_SCORE_INFINITY,
 486                           PCMK_META_TARGET_ROLE, scheduler);
 487 
 488     } else if ((rsc->priv->next_role > rsc->priv->orig_role)
 489                && !pcmk_is_set(scheduler->flags, pcmk__sched_quorate)
 490                && (scheduler->no_quorum_policy == pcmk_no_quorum_freeze)) {
 491         crm_notice("Resource %s cannot be elevated from %s to %s due to "
 492                    PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE,
 493                    rsc->id, pcmk_role_text(rsc->priv->orig_role),
 494                    pcmk_role_text(rsc->priv->next_role));
 495         pe__set_next_role(rsc, rsc->priv->orig_role,
 496                           PCMK_OPT_NO_QUORUM_POLICY "=" PCMK_VALUE_FREEZE);
 497     }
 498 
 499     pe__show_node_scores(!pcmk_is_set(scheduler->flags,
 500                                       pcmk__sched_output_scores),
 501                          rsc, __func__, rsc->priv->allowed_nodes, scheduler);
 502 
 503     // Unmanage resource if fencing is enabled but no device is configured
 504     if (pcmk_is_set(scheduler->flags, pcmk__sched_fencing_enabled)
 505         && !pcmk_is_set(scheduler->flags, pcmk__sched_have_fencing)) {
 506         pcmk__clear_rsc_flags(rsc, pcmk__rsc_managed);
 507     }
 508 
 509     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 510         // Unmanaged resources stay on their current node
 511         const char *reason = NULL;
 512         pcmk_node_t *assign_to = NULL;
 513 
 514         pe__set_next_role(rsc, rsc->priv->orig_role, "unmanaged");
 515         assign_to = pcmk__current_node(rsc);
 516         if (assign_to == NULL) {
 517             reason = "inactive";
 518         } else if (rsc->priv->orig_role == pcmk_role_promoted) {
 519             reason = "promoted";
 520         } else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
 521             reason = "failed";
 522         } else {
 523             reason = "active";
 524         }
 525         pcmk__rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
 526                        (assign_to? assign_to->priv->name : "no node"),
 527                        reason);
 528         pcmk__assign_resource(rsc, assign_to, true, stop_if_fail);
 529 
 530     } else if (pcmk_is_set(scheduler->flags, pcmk__sched_stop_all)) {
 531         // Must stop at some point, but be consistent with stop_if_fail
 532         if (stop_if_fail) {
 533             pcmk__rsc_debug(rsc,
 534                             "Forcing %s to stop: " PCMK_OPT_STOP_ALL_RESOURCES,
 535                             rsc->id);
 536         }
 537         pcmk__assign_resource(rsc, NULL, true, stop_if_fail);
 538 
 539     } else if (!assign_best_node(rsc, prefer, stop_if_fail)) {
 540         // Assignment failed
 541         if (!pcmk_is_set(rsc->flags, pcmk__rsc_removed)) {
 542             pcmk__rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
 543         } else if ((rsc->priv->active_nodes != NULL) && stop_if_fail) {
 544             pcmk__rsc_info(rsc, "Stopping removed resource %s", rsc->id);
 545         }
 546     }
 547 
 548     pcmk__clear_rsc_flags(rsc, pcmk__rsc_assigning);
 549 
 550     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
 551         remote_connection_assigned(rsc);
 552     }
 553 
 554     return rsc->priv->assigned_node;
 555 }
 556 
 557 /*!
 558  * \internal
 559  * \brief Schedule actions to bring resource down and back to current role
 560  *
 561  * \param[in,out] rsc           Resource to restart
 562  * \param[in,out] current       Node that resource should be brought down on
 563  * \param[in]     need_stop     Whether the resource must be stopped
 564  * \param[in]     need_promote  Whether the resource must be promoted
 565  *
 566  * \return Role that resource would have after scheduled actions are taken
 567  */
 568 static void
 569 schedule_restart_actions(pcmk_resource_t *rsc, pcmk_node_t *current,
     /*  */
 570                          bool need_stop, bool need_promote)
 571 {
 572     enum rsc_role_e role = rsc->priv->orig_role;
 573     enum rsc_role_e next_role;
 574     rsc_transition_fn fn = NULL;
 575 
 576     pcmk__set_rsc_flags(rsc, pcmk__rsc_restarting);
 577 
 578     // Bring resource down to a stop on its current node
 579     while (role != pcmk_role_stopped) {
 580         next_role = rsc_state_matrix[role][pcmk_role_stopped];
 581         pcmk__rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
 582                         (need_stop? "required" : "optional"), rsc->id,
 583                         pcmk_role_text(role), pcmk_role_text(next_role));
 584         fn = rsc_action_matrix[role][next_role];
 585         if (fn == NULL) {
 586             break;
 587         }
 588         fn(rsc, current, !need_stop);
 589         role = next_role;
 590     }
 591 
 592     // Bring resource up to its next role on its next node
 593     while ((rsc->priv->orig_role <= rsc->priv->next_role)
 594            && (role != rsc->priv->orig_role)
 595            && !pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
 596         bool required = need_stop;
 597 
 598         next_role = rsc_state_matrix[role][rsc->priv->orig_role];
 599         if ((next_role == pcmk_role_promoted) && need_promote) {
 600             required = true;
 601         }
 602         pcmk__rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
 603                         (required? "required" : "optional"), rsc->id,
 604                         pcmk_role_text(role), pcmk_role_text(next_role));
 605         fn = rsc_action_matrix[role][next_role];
 606         if (fn == NULL) {
 607             break;
 608         }
 609         fn(rsc, rsc->priv->assigned_node, !required);
 610         role = next_role;
 611     }
 612 
 613     pcmk__clear_rsc_flags(rsc, pcmk__rsc_restarting);
 614 }
 615 
 616 /*!
 617  * \internal
 618  * \brief If a resource's next role is not explicitly specified, set a default
 619  *
 620  * \param[in,out] rsc  Resource to set next role for
 621  *
 622  * \return "explicit" if next role was explicitly set, otherwise "implicit"
 623  */
 624 static const char *
 625 set_default_next_role(pcmk_resource_t *rsc)
     /*  */
 626 {
 627     if (rsc->priv->next_role != pcmk_role_unknown) {
 628         return "explicit";
 629     }
 630 
 631     if (rsc->priv->assigned_node == NULL) {
 632         pe__set_next_role(rsc, pcmk_role_stopped, "assignment");
 633     } else {
 634         pe__set_next_role(rsc, pcmk_role_started, "assignment");
 635     }
 636     return "implicit";
 637 }
 638 
 639 /*!
 640  * \internal
 641  * \brief Create an action to represent an already pending start
 642  *
 643  * \param[in,out] rsc  Resource to create start action for
 644  */
 645 static void
 646 create_pending_start(pcmk_resource_t *rsc)
     /*  */
 647 {
 648     pcmk_action_t *start = NULL;
 649 
 650     pcmk__rsc_trace(rsc,
 651                     "Creating action for %s to represent already pending start",
 652                     rsc->id);
 653     start = start_action(rsc, rsc->priv->assigned_node, TRUE);
 654     pcmk__set_action_flags(start, pcmk__action_always_in_graph);
 655 }
 656 
 657 /*!
 658  * \internal
 659  * \brief Schedule actions needed to take a resource to its next role
 660  *
 661  * \param[in,out] rsc  Resource to schedule actions for
 662  */
 663 static void
 664 schedule_role_transition_actions(pcmk_resource_t *rsc)
     /*  */
 665 {
 666     enum rsc_role_e role = rsc->priv->orig_role;
 667 
 668     while (role != rsc->priv->next_role) {
 669         enum rsc_role_e next_role =
 670             rsc_state_matrix[role][rsc->priv->next_role];
 671         rsc_transition_fn fn = NULL;
 672 
 673         pcmk__rsc_trace(rsc,
 674                         "Creating action to take %s from %s to %s "
 675                         "(ending at %s)",
 676                         rsc->id, pcmk_role_text(role),
 677                         pcmk_role_text(next_role),
 678                         pcmk_role_text(rsc->priv->next_role));
 679         fn = rsc_action_matrix[role][next_role];
 680         if (fn == NULL) {
 681             break;
 682         }
 683         fn(rsc, rsc->priv->assigned_node, false);
 684         role = next_role;
 685     }
 686 }
 687 
 688 /*!
 689  * \internal
 690  * \brief Create all actions needed for a given primitive resource
 691  *
 692  * \param[in,out] rsc  Primitive resource to create actions for
 693  */
 694 void
 695 pcmk__primitive_create_actions(pcmk_resource_t *rsc)
     /*  */
 696 {
 697     bool need_stop = false;
 698     bool need_promote = false;
 699     bool is_moving = false;
 700     bool allow_migrate = false;
 701     bool multiply_active = false;
 702 
 703     pcmk_node_t *current = NULL;
 704     pcmk_node_t *migration_target = NULL;
 705     unsigned int num_all_active = 0;
 706     unsigned int num_clean_active = 0;
 707     const char *next_role_source = NULL;
 708 
 709     pcmk__assert(pcmk__is_primitive(rsc));
 710 
 711     next_role_source = set_default_next_role(rsc);
 712     pcmk__rsc_trace(rsc,
 713                     "Creating all actions for %s transition from %s to %s "
 714                     "(%s) on %s",
 715                     rsc->id, pcmk_role_text(rsc->priv->orig_role),
 716                     pcmk_role_text(rsc->priv->next_role), next_role_source,
 717                     pcmk__node_name(rsc->priv->assigned_node));
 718 
 719     current = rsc->priv->fns->active_node(rsc, &num_all_active,
 720                                           &num_clean_active);
 721 
 722     g_list_foreach(rsc->priv->dangling_migration_sources,
 723                    pcmk__abort_dangling_migration, rsc);
 724 
 725     if ((current != NULL) && (rsc->priv->assigned_node != NULL)
 726         && !pcmk__same_node(current, rsc->priv->assigned_node)
 727         && (rsc->priv->next_role >= pcmk_role_started)) {
 728 
 729         pcmk__rsc_trace(rsc, "Moving %s from %s to %s",
 730                         rsc->id, pcmk__node_name(current),
 731                         pcmk__node_name(rsc->priv->assigned_node));
 732         is_moving = true;
 733         allow_migrate = pcmk__rsc_can_migrate(rsc, current);
 734 
 735         // This is needed even if migrating (though I'm not sure why ...)
 736         need_stop = true;
 737     }
 738 
 739     // Check whether resource is partially migrated and/or multiply active
 740     migration_target = rsc->priv->partial_migration_target;
 741     if ((rsc->priv->partial_migration_source != NULL)
 742         && (migration_target != NULL) && allow_migrate && (num_all_active == 2)
 743         && pcmk__same_node(current, rsc->priv->partial_migration_source)
 744         && pcmk__same_node(rsc->priv->assigned_node, migration_target)) {
 745         /* A partial migration is in progress, and the migration target remains
 746          * the same as when the migration began.
 747          */
 748         pcmk__rsc_trace(rsc,
 749                         "Partial migration of %s from %s to %s will continue",
 750                         rsc->id,
 751                         pcmk__node_name(rsc->priv->partial_migration_source),
 752                         pcmk__node_name(migration_target));
 753 
 754     } else if ((rsc->priv->partial_migration_source != NULL)
 755                || (migration_target != NULL)) {
 756         // A partial migration is in progress but can't be continued
 757 
 758         if (num_all_active > 2) {
 759             // The resource is migrating *and* multiply active!
 760             crm_notice("Forcing recovery of %s because it is migrating "
 761                        "from %s to %s and possibly active elsewhere",
 762                        rsc->id,
 763                        pcmk__node_name(rsc->priv->partial_migration_source),
 764                        pcmk__node_name(migration_target));
 765         } else {
 766             // The migration source or target isn't available
 767             crm_notice("Forcing recovery of %s because it can no longer "
 768                        "migrate from %s to %s",
 769                        rsc->id,
 770                        pcmk__node_name(rsc->priv->partial_migration_source),
 771                        pcmk__node_name(migration_target));
 772         }
 773         need_stop = true;
 774         rsc->priv->partial_migration_source = NULL;
 775         rsc->priv->partial_migration_target = NULL;
 776         allow_migrate = false;
 777 
 778     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_fencing)) {
 779         multiply_active = (num_all_active > 1);
 780     } else {
 781         /* If a resource has PCMK_META_REQUIRES set to PCMK_VALUE_NOTHING or
 782          * PCMK_VALUE_QUORUM, don't consider it active on unclean nodes (similar
 783          * to how all resources behave when PCMK_OPT_STONITH_ENABLED is false).
 784          * We can start such resources elsewhere before fencing completes, and
 785          * if we considered the resource active on the failed node, we would
 786          * attempt recovery for being active on multiple nodes.
 787          */
 788         multiply_active = (num_clean_active > 1);
 789     }
 790 
 791     if (multiply_active) {
 792         const char *class = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
 793 
 794         // Resource was (possibly) incorrectly multiply active
 795         pcmk__sched_err(rsc->priv->scheduler,
 796                         "%s resource %s might be active on %u nodes (%s)",
 797                         pcmk__s(class, "Untyped"), rsc->id, num_all_active,
 798                         pcmk__multiply_active_text(rsc));
 799         crm_notice("For more information, see \"What are multiply active "
 800                    "resources?\" at "
 801                    "https://projects.clusterlabs.org/w/clusterlabs/faq/");
 802 
 803         switch (rsc->priv->multiply_active_policy) {
 804             case pcmk__multiply_active_restart:
 805                 need_stop = true;
 806                 break;
 807             case pcmk__multiply_active_unexpected:
 808                 need_stop = true; // stop_resource() will skip expected node
 809                 pcmk__set_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
 810                 break;
 811             default:
 812                 break;
 813         }
 814 
 815     } else {
 816         pcmk__clear_rsc_flags(rsc, pcmk__rsc_stop_unexpected);
 817     }
 818 
 819     if (pcmk_is_set(rsc->flags, pcmk__rsc_start_pending)) {
 820         create_pending_start(rsc);
 821     }
 822 
 823     if (is_moving) {
 824         // Remaining tests are only for resources staying where they are
 825 
 826     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
 827         if (pcmk_is_set(rsc->flags, pcmk__rsc_stop_if_failed)) {
 828             need_stop = true;
 829             pcmk__rsc_trace(rsc, "Recovering %s", rsc->id);
 830         } else {
 831             pcmk__rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
 832             if (rsc->priv->next_role == pcmk_role_promoted) {
 833                 need_promote = true;
 834             }
 835         }
 836 
 837     } else if (pcmk_is_set(rsc->flags, pcmk__rsc_blocked)) {
 838         pcmk__rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
 839         need_stop = true;
 840 
 841     } else if ((rsc->priv->orig_role > pcmk_role_started)
 842                && (current != NULL)
 843                && (rsc->priv->assigned_node != NULL)) {
 844         pcmk_action_t *start = NULL;
 845 
 846         pcmk__rsc_trace(rsc, "Creating start action for promoted resource %s",
 847                         rsc->id);
 848         start = start_action(rsc, rsc->priv->assigned_node, TRUE);
 849         if (!pcmk_is_set(start->flags, pcmk__action_optional)) {
 850             // Recovery of a promoted resource
 851             pcmk__rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
 852             need_stop = true;
 853         }
 854     }
 855 
 856     // Create any actions needed to bring resource down and back up to same role
 857     schedule_restart_actions(rsc, current, need_stop, need_promote);
 858 
 859     // Create any actions needed to take resource from this role to the next
 860     schedule_role_transition_actions(rsc);
 861 
 862     pcmk__create_recurring_actions(rsc);
 863 
 864     if (allow_migrate) {
 865         pcmk__create_migration_actions(rsc, current);
 866     }
 867 }
 868 
 869 /*!
 870  * \internal
 871  * \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
 872  *
 873  * \param[in] rsc  Resource to check
 874  */
 875 static void
 876 rsc_avoids_remote_nodes(const pcmk_resource_t *rsc)
     /*  */
 877 {
 878     GHashTableIter iter;
 879     pcmk_node_t *node = NULL;
 880 
 881     g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes);
 882     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
 883         if (node->priv->remote != NULL) {
 884             node->assign->score = -PCMK_SCORE_INFINITY;
 885         }
 886     }
 887 }
 888 
 889 /*!
 890  * \internal
 891  * \brief Return allowed nodes as (possibly sorted) list
 892  *
 893  * Convert a resource's hash table of allowed nodes to a list. If printing to
 894  * stdout, sort the list, to keep action ID numbers consistent for regression
 895  * test output (while avoiding the performance hit on a live cluster).
 896  *
 897  * \param[in] rsc       Resource to check for allowed nodes
 898  *
 899  * \return List of resource's allowed nodes
 900  * \note Callers should take care not to rely on the list being sorted.
 901  */
 902 static GList *
 903 allowed_nodes_as_list(const pcmk_resource_t *rsc)
     /*  */
 904 {
 905     GList *allowed_nodes = NULL;
 906 
 907     if (rsc->priv->allowed_nodes != NULL) {
 908         allowed_nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
 909     }
 910 
 911     if (!pcmk__is_daemon) {
 912         allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
 913     }
 914 
 915     return allowed_nodes;
 916 }
 917 
 918 /*!
 919  * \internal
 920  * \brief Create implicit constraints needed for a primitive resource
 921  *
 922  * \param[in,out] rsc  Primitive resource to create implicit constraints for
 923  */
 924 void
 925 pcmk__primitive_internal_constraints(pcmk_resource_t *rsc)
     /*  */
 926 {
 927     GList *allowed_nodes = NULL;
 928     bool check_unfencing = false;
 929     bool check_utilization = false;
 930     pcmk_scheduler_t *scheduler = NULL;
 931 
 932     pcmk__assert(pcmk__is_primitive(rsc));
 933     scheduler = rsc->priv->scheduler;
 934 
 935     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
 936         pcmk__rsc_trace(rsc,
 937                         "Skipping implicit constraints for unmanaged resource "
 938                         "%s", rsc->id);
 939         return;
 940     }
 941 
 942     // Whether resource requires unfencing
 943     check_unfencing = !pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)
 944                       && pcmk_is_set(scheduler->flags,
 945                                      pcmk__sched_enable_unfencing)
 946                       && pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing);
 947 
 948     // Whether a non-default placement strategy is used
 949     check_utilization = (g_hash_table_size(rsc->priv->utilization) > 0)
 950                          && !pcmk__str_eq(scheduler->priv->placement_strategy,
 951                                           PCMK_VALUE_DEFAULT, pcmk__str_casei);
 952 
 953     // Order stops before starts (i.e. restart)
 954     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0), NULL,
 955                        rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0), NULL,
 956                        pcmk__ar_ordered
 957                        |pcmk__ar_first_implies_then
 958                        |pcmk__ar_intermediate_stop, scheduler);
 959 
 960     // Promotable ordering: demote before stop, start before promote
 961     if (pcmk_is_set(pe__const_top_resource(rsc, false)->flags,
 962                     pcmk__rsc_promotable)
 963         || (rsc->priv->orig_role > pcmk_role_unpromoted)) {
 964 
 965         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_DEMOTE, 0),
 966                            NULL,
 967                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
 968                            NULL,
 969                            pcmk__ar_promoted_then_implies_first, scheduler);
 970 
 971         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
 972                            NULL,
 973                            rsc, pcmk__op_key(rsc->id, PCMK_ACTION_PROMOTE, 0),
 974                            NULL,
 975                            pcmk__ar_unrunnable_first_blocks, scheduler);
 976     }
 977 
 978     // Don't clear resource history if probing on same node
 979     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, PCMK_ACTION_LRM_DELETE, 0),
 980                        NULL, rsc,
 981                        pcmk__op_key(rsc->id, PCMK_ACTION_MONITOR, 0),
 982                        NULL,
 983                        pcmk__ar_if_on_same_node|pcmk__ar_then_cancels_first,
 984                        scheduler);
 985 
 986     // Certain checks need allowed nodes
 987     if (check_unfencing || check_utilization
 988         || (rsc->priv->launcher != NULL)) {
 989 
 990         allowed_nodes = allowed_nodes_as_list(rsc);
 991     }
 992 
 993     if (check_unfencing) {
 994         g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
 995     }
 996 
 997     if (check_utilization) {
 998         pcmk__create_utilization_constraints(rsc, allowed_nodes);
 999     }
1000 
1001     if (rsc->priv->launcher != NULL) {
1002         pcmk_resource_t *remote_rsc = NULL;
1003 
1004         if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)) {
1005             // rsc is the implicit remote connection for a guest or bundle node
1006 
1007             /* Guest resources are not allowed to run on Pacemaker Remote nodes,
1008              * to avoid nesting remotes. However, bundles are allowed.
1009              */
1010             if (!pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)) {
1011                 rsc_avoids_remote_nodes(rsc->priv->launcher);
1012             }
1013 
1014             /* If someone cleans up a guest or bundle node's launcher, we will
1015              * likely schedule a (re-)probe of the launcher and recovery of the
1016              * connection. Order the connection stop after the launcher probe,
1017              * so that if we detect the launcher running, we will trigger a new
1018              * transition and avoid the unnecessary recovery.
1019              */
1020             pcmk__order_resource_actions(rsc->priv->launcher,
1021                                          PCMK_ACTION_MONITOR,
1022                                          rsc, PCMK_ACTION_STOP,
1023                                          pcmk__ar_ordered);
1024 
1025         /* A user can specify that a resource must start on a Pacemaker Remote
1026          * node by explicitly configuring it with the PCMK__META_CONTAINER
1027          * meta-attribute. This is of questionable merit, since location
1028          * constraints can accomplish the same thing. But we support it, so here
1029          * we check whether a resource (that is not itself a remote connection)
1030          * has PCMK__META_CONTAINER set to a remote node or guest node resource.
1031          */
1032         } else if (pcmk_is_set(rsc->priv->launcher->flags,
1033                                pcmk__rsc_is_remote_connection)) {
1034             remote_rsc = rsc->priv->launcher;
1035         } else  {
1036             remote_rsc =
1037                 pe__resource_contains_guest_node(scheduler,
1038                                                  rsc->priv->launcher);
1039         }
1040 
1041         if (remote_rsc != NULL) {
1042             /* Force the resource on the Pacemaker Remote node instead of
1043              * colocating the resource with the launcher.
1044              */
1045             for (GList *item = allowed_nodes; item; item = item->next) {
1046                 pcmk_node_t *node = item->data;
1047 
1048                 if (node->priv->remote != remote_rsc) {
1049                     node->assign->score = -PCMK_SCORE_INFINITY;
1050                 }
1051             }
1052 
1053         } else {
1054             /* This resource is either launched by a resource that does NOT
1055              * represent a Pacemaker Remote node, or a Pacemaker Remote
1056              * connection resource for a guest node or bundle.
1057              */
1058             int score;
1059 
1060             crm_trace("Order and colocate %s relative to its launcher %s",
1061                       rsc->id, rsc->priv->launcher->id);
1062 
1063             pcmk__new_ordering(rsc->priv->launcher,
1064                                pcmk__op_key(rsc->priv->launcher->id,
1065                                             PCMK_ACTION_START, 0),
1066                                NULL, rsc,
1067                                pcmk__op_key(rsc->id, PCMK_ACTION_START, 0),
1068                                NULL,
1069                                pcmk__ar_first_implies_then
1070                                |pcmk__ar_unrunnable_first_blocks, scheduler);
1071 
1072             pcmk__new_ordering(rsc,
1073                                pcmk__op_key(rsc->id, PCMK_ACTION_STOP, 0),
1074                                NULL,
1075                                rsc->priv->launcher,
1076                                pcmk__op_key(rsc->priv->launcher->id,
1077                                             PCMK_ACTION_STOP, 0),
1078                                NULL, pcmk__ar_then_implies_first, scheduler);
1079 
1080             if (pcmk_is_set(rsc->flags, pcmk__rsc_remote_nesting_allowed)
1081                 /* @TODO: && non-bundle Pacemaker Remote nodes exist */) {
1082                 score = 10000;    /* Highly preferred but not essential */
1083             } else {
1084                 score = PCMK_SCORE_INFINITY; // Force to run on same host
1085             }
1086             pcmk__new_colocation("#resource-with-container", NULL, score, rsc,
1087                                  rsc->priv->launcher, NULL, NULL,
1088                                  pcmk__coloc_influence);
1089         }
1090     }
1091 
1092     if (pcmk_is_set(rsc->flags, pcmk__rsc_is_remote_connection)
1093         || pcmk_is_set(rsc->flags, pcmk__rsc_fence_device)) {
1094         /* Remote connections and fencing devices are not allowed to run on
1095          * Pacemaker Remote nodes
1096          */
1097         rsc_avoids_remote_nodes(rsc);
1098     }
1099     g_list_free(allowed_nodes);
1100 }
1101 
1102 /*!
1103  * \internal
1104  * \brief Apply a colocation's score to node scores or resource priority
1105  *
1106  * Given a colocation constraint, apply its score to the dependent's
1107  * allowed node scores (if we are still placing resources) or priority (if
1108  * we are choosing promotable clone instance roles).
1109  *
1110  * \param[in,out] dependent      Dependent resource in colocation
1111  * \param[in]     primary        Primary resource in colocation
1112  * \param[in]     colocation     Colocation constraint to apply
1113  * \param[in]     for_dependent  true if called on behalf of dependent
1114  *
1115  * \return The score added to the dependent's priority
1116  */
1117 int
1118 pcmk__primitive_apply_coloc_score(pcmk_resource_t *dependent,
     /*  */
1119                                   const pcmk_resource_t *primary,
1120                                   const pcmk__colocation_t *colocation,
1121                                   bool for_dependent)
1122 {
1123     enum pcmk__coloc_affects filter_results;
1124 
1125     pcmk__assert((dependent != NULL) && (primary != NULL)
1126                  && (colocation != NULL));
1127 
1128     if (for_dependent) {
1129         // Always process on behalf of primary resource
1130         return primary->priv->cmds->apply_coloc_score(dependent, primary,
1131                                                       colocation, false);
1132     }
1133 
1134     filter_results = pcmk__colocation_affects(dependent, primary, colocation,
1135                                               false);
1136     pcmk__rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
1137                     ((colocation->score > 0)? "Colocating" : "Anti-colocating"),
1138                     dependent->id, primary->id, colocation->id,
1139                     colocation->score,
1140                     filter_results);
1141 
1142     switch (filter_results) {
1143         case pcmk__coloc_affects_role:
1144             return pcmk__apply_coloc_to_priority(dependent, primary,
1145                                                  colocation);
1146 
1147         case pcmk__coloc_affects_location:
1148             pcmk__apply_coloc_to_scores(dependent, primary, colocation);
1149             return 0;
1150 
1151         default: // pcmk__coloc_affects_nothing
1152             return 0;
1153     }
1154 }
1155 
1156 /* Primitive implementation of
1157  * pcmk__assignment_methods_t:with_this_colocations()
1158  */
1159 void
1160 pcmk__with_primitive_colocations(const pcmk_resource_t *rsc,
     /*  */
1161                                  const pcmk_resource_t *orig_rsc, GList **list)
1162 {
1163     const pcmk_resource_t *parent = NULL;
1164 
1165     pcmk__assert(pcmk__is_primitive(rsc) && (list != NULL));
1166     parent = rsc->priv->parent;
1167 
1168     if (rsc == orig_rsc) {
1169         /* For the resource itself, add all of its own colocations and relevant
1170          * colocations from its parent (if any).
1171          */
1172         pcmk__add_with_this_list(list, rsc->priv->with_this_colocations,
1173                                  orig_rsc);
1174         if (parent != NULL) {
1175             parent->priv->cmds->with_this_colocations(parent, orig_rsc, list);
1176         }
1177     } else {
1178         // For an ancestor, add only explicitly configured constraints
1179         for (GList *iter = rsc->priv->with_this_colocations;
1180              iter != NULL; iter = iter->next) {
1181             pcmk__colocation_t *colocation = iter->data;
1182 
1183             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
1184                 pcmk__add_with_this(list, colocation, orig_rsc);
1185             }
1186         }
1187     }
1188 }
1189 
1190 /* Primitive implementation of
1191  * pcmk__assignment_methods_t:this_with_colocations()
1192  */
1193 void
1194 pcmk__primitive_with_colocations(const pcmk_resource_t *rsc,
     /*  */
1195                                  const pcmk_resource_t *orig_rsc, GList **list)
1196 {
1197     const pcmk_resource_t *parent = NULL;
1198 
1199     pcmk__assert(pcmk__is_primitive(rsc) && (list != NULL));
1200     parent = rsc->priv->parent;
1201 
1202     if (rsc == orig_rsc) {
1203         /* For the resource itself, add all of its own colocations and relevant
1204          * colocations from its parent (if any).
1205          */
1206         pcmk__add_this_with_list(list, rsc->priv->this_with_colocations,
1207                                  orig_rsc);
1208         if (parent != NULL) {
1209             parent->priv->cmds->this_with_colocations(parent, orig_rsc, list);
1210         }
1211     } else {
1212         // For an ancestor, add only explicitly configured constraints
1213         for (GList *iter = rsc->priv->this_with_colocations;
1214              iter != NULL; iter = iter->next) {
1215             pcmk__colocation_t *colocation = iter->data;
1216 
1217             if (pcmk_is_set(colocation->flags, pcmk__coloc_explicit)) {
1218                 pcmk__add_this_with(list, colocation, orig_rsc);
1219             }
1220         }
1221     }
1222 }
1223 
1224 /*!
1225  * \internal
1226  * \brief Return action flags for a given primitive resource action
1227  *
1228  * \param[in,out] action  Action to get flags for
1229  * \param[in]     node    If not NULL, limit effects to this node (ignored)
1230  *
1231  * \return Flags appropriate to \p action on \p node
1232  */
1233 uint32_t
1234 pcmk__primitive_action_flags(pcmk_action_t *action, const pcmk_node_t *node)
     /*  */
1235 {
1236     pcmk__assert(action != NULL);
1237     return (uint32_t) action->flags;
1238 }
1239 
1240 /*!
1241  * \internal
1242  * \brief Check whether a node is a multiply active resource's expected node
1243  *
1244  * \param[in] rsc  Resource to check
1245  * \param[in] node  Node to check
1246  *
1247  * \return \c true if \p rsc is multiply active with
1248  *         \c PCMK_META_MULTIPLE_ACTIVE set to \c PCMK_VALUE_STOP_UNEXPECTED,
1249  *         and \p node is the node where it will remain active
1250  * \note This assumes that the resource's next role cannot be changed to stopped
1251  *       after this is called, which should be reasonable if status has already
1252  *       been unpacked and resources have been assigned to nodes.
1253  */
1254 static bool
1255 is_expected_node(const pcmk_resource_t *rsc, const pcmk_node_t *node)
     /*  */
1256 {
1257     return pcmk_all_flags_set(rsc->flags,
1258                               pcmk__rsc_stop_unexpected|pcmk__rsc_restarting)
1259            && (rsc->priv->next_role > pcmk_role_stopped)
1260            && pcmk__same_node(rsc->priv->assigned_node, node);
1261 }
1262 
1263 /*!
1264  * \internal
1265  * \brief Schedule actions needed to stop a resource wherever it is active
1266  *
1267  * \param[in,out] rsc       Resource being stopped
1268  * \param[in]     node      Node where resource is being stopped (ignored)
1269  * \param[in]     optional  Whether actions should be optional
1270  */
1271 static void
1272 stop_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
     /*  */
1273 {
1274     for (GList *iter = rsc->priv->active_nodes;
1275          iter != NULL; iter = iter->next) {
1276 
1277         pcmk_node_t *current = (pcmk_node_t *) iter->data;
1278         pcmk_action_t *stop = NULL;
1279 
1280         if (is_expected_node(rsc, current)) {
1281             /* We are scheduling restart actions for a multiply active resource
1282              * with PCMK_META_MULTIPLE_ACTIVE=PCMK_VALUE_STOP_UNEXPECTED, and
1283              * this is where it should not be stopped.
1284              */
1285             pcmk__rsc_trace(rsc,
1286                             "Skipping stop of multiply active resource %s "
1287                             "on expected node %s",
1288                             rsc->id, pcmk__node_name(current));
1289             continue;
1290         }
1291 
1292         if (rsc->priv->partial_migration_target != NULL) {
1293             // Continue migration if node originally was and remains target
1294             if (pcmk__same_node(current, rsc->priv->partial_migration_target)
1295                 && pcmk__same_node(current, rsc->priv->assigned_node)) {
1296                 pcmk__rsc_trace(rsc,
1297                                 "Skipping stop of %s on %s "
1298                                 "because partial migration there will continue",
1299                                 rsc->id, pcmk__node_name(current));
1300                 continue;
1301             } else {
1302                 pcmk__rsc_trace(rsc,
1303                                 "Forcing stop of %s on %s "
1304                                 "because migration target changed",
1305                                 rsc->id, pcmk__node_name(current));
1306                 optional = false;
1307             }
1308         }
1309 
1310         pcmk__rsc_trace(rsc, "Scheduling stop of %s on %s",
1311                         rsc->id, pcmk__node_name(current));
1312         stop = stop_action(rsc, current, optional);
1313 
1314         if (rsc->priv->assigned_node == NULL) {
1315             pe_action_set_reason(stop, "node availability", true);
1316         } else if (pcmk_all_flags_set(rsc->flags, pcmk__rsc_restarting
1317                                                   |pcmk__rsc_stop_unexpected)) {
1318             /* We are stopping a multiply active resource on a node that is
1319              * not its expected node, and we are still scheduling restart
1320              * actions, so the stop is for being multiply active.
1321              */
1322             pe_action_set_reason(stop, "being multiply active", true);
1323         }
1324 
1325         if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
1326             pcmk__clear_action_flags(stop, pcmk__action_runnable);
1327         }
1328 
1329         if (pcmk_is_set(rsc->flags, pcmk__rsc_needs_unfencing)) {
1330             pcmk_action_t *unfence = pe_fence_op(current, PCMK_ACTION_ON, true,
1331                                                  NULL, false,
1332                                                  rsc->priv->scheduler);
1333 
1334             order_actions(stop, unfence, pcmk__ar_then_implies_first);
1335             if (!pcmk__node_unfenced(current)) {
1336                 pcmk__sched_err(rsc->priv->scheduler,
1337                                 "Stopping %s until %s can be unfenced",
1338                                 rsc->id, pcmk__node_name(current));
1339             }
1340         }
1341     }
1342 }
1343 
1344 /*!
1345  * \internal
1346  * \brief Schedule actions needed to start a resource on a node
1347  *
1348  * \param[in,out] rsc       Resource being started
1349  * \param[in,out] node      Node where resource should be started
1350  * \param[in]     optional  Whether actions should be optional
1351  */
1352 static void
1353 start_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
     /*  */
1354 {
1355     pcmk_action_t *start = NULL;
1356 
1357     pcmk__assert(node != NULL);
1358 
1359     pcmk__rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
1360                     (optional? "optional" : "required"), rsc->id,
1361                     pcmk__node_name(node), node->assign->score);
1362     start = start_action(rsc, node, TRUE);
1363 
1364     pcmk__order_vs_unfence(rsc, node, start, pcmk__ar_first_implies_then);
1365 
1366     if (pcmk_is_set(start->flags, pcmk__action_runnable) && !optional) {
1367         pcmk__clear_action_flags(start, pcmk__action_optional);
1368     }
1369 
1370     if (is_expected_node(rsc, node)) {
1371         /* This could be a problem if the start becomes necessary for other
1372          * reasons later.
1373          */
1374         pcmk__rsc_trace(rsc,
1375                         "Start of multiply active resouce %s "
1376                         "on expected node %s will be a pseudo-action",
1377                         rsc->id, pcmk__node_name(node));
1378         pcmk__set_action_flags(start, pcmk__action_pseudo);
1379     }
1380 }
1381 
1382 /*!
1383  * \internal
1384  * \brief Schedule actions needed to promote a resource on a node
1385  *
1386  * \param[in,out] rsc       Resource being promoted
1387  * \param[in]     node      Node where resource should be promoted
1388  * \param[in]     optional  Whether actions should be optional
1389  */
1390 static void
1391 promote_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
     /*  */
1392 {
1393     GList *iter = NULL;
1394     GList *action_list = NULL;
1395     bool runnable = true;
1396 
1397     pcmk__assert(node != NULL);
1398 
1399     // Any start must be runnable for promotion to be runnable
1400     action_list = pe__resource_actions(rsc, node, PCMK_ACTION_START, true);
1401     for (iter = action_list; iter != NULL; iter = iter->next) {
1402         pcmk_action_t *start = (pcmk_action_t *) iter->data;
1403 
1404         if (!pcmk_is_set(start->flags, pcmk__action_runnable)) {
1405             runnable = false;
1406         }
1407     }
1408     g_list_free(action_list);
1409 
1410     if (runnable) {
1411         pcmk_action_t *promote = promote_action(rsc, node, optional);
1412 
1413         pcmk__rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
1414                         (optional? "optional" : "required"), rsc->id,
1415                         pcmk__node_name(node));
1416 
1417         if (is_expected_node(rsc, node)) {
1418             /* This could be a problem if the promote becomes necessary for
1419              * other reasons later.
1420              */
1421             pcmk__rsc_trace(rsc,
1422                             "Promotion of multiply active resouce %s "
1423                             "on expected node %s will be a pseudo-action",
1424                             rsc->id, pcmk__node_name(node));
1425             pcmk__set_action_flags(promote, pcmk__action_pseudo);
1426         }
1427     } else {
1428         pcmk__rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
1429                         rsc->id, pcmk__node_name(node));
1430         action_list = pe__resource_actions(rsc, node, PCMK_ACTION_PROMOTE,
1431                                            true);
1432         for (iter = action_list; iter != NULL; iter = iter->next) {
1433             pcmk_action_t *promote = (pcmk_action_t *) iter->data;
1434 
1435             pcmk__clear_action_flags(promote, pcmk__action_runnable);
1436         }
1437         g_list_free(action_list);
1438     }
1439 }
1440 
1441 /*!
1442  * \internal
1443  * \brief Schedule actions needed to demote a resource wherever it is active
1444  *
1445  * \param[in,out] rsc       Resource being demoted
1446  * \param[in]     node      Node where resource should be demoted (ignored)
1447  * \param[in]     optional  Whether actions should be optional
1448  */
1449 static void
1450 demote_resource(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
     /*  */
1451 {
1452     /* Since this will only be called for a primitive (possibly as an instance
1453      * of a collective resource), the resource is multiply active if it is
1454      * running on more than one node, so we want to demote on all of them as
1455      * part of recovery, regardless of which one is the desired node.
1456      */
1457     for (GList *iter = rsc->priv->active_nodes;
1458          iter != NULL; iter = iter->next) {
1459 
1460         pcmk_node_t *current = (pcmk_node_t *) iter->data;
1461 
1462         if (is_expected_node(rsc, current)) {
1463             pcmk__rsc_trace(rsc,
1464                             "Skipping demote of multiply active resource %s "
1465                             "on expected node %s",
1466                             rsc->id, pcmk__node_name(current));
1467         } else {
1468             pcmk__rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
1469                             (optional? "optional" : "required"), rsc->id,
1470                             pcmk__node_name(current));
1471             demote_action(rsc, current, optional);
1472         }
1473     }
1474 }
1475 
1476 static void
1477 assert_role_error(pcmk_resource_t *rsc, pcmk_node_t *node, bool optional)
     /*  */
1478 {
1479     pcmk__assert(false);
1480 }
1481 
1482 /*!
1483  * \internal
1484  * \brief Schedule cleanup of a resource
1485  *
1486  * \param[in,out] rsc       Resource to clean up
1487  * \param[in]     node      Node to clean up on
1488  * \param[in]     optional  Whether clean-up should be optional
1489  */
1490 void
1491 pcmk__schedule_cleanup(pcmk_resource_t *rsc, const pcmk_node_t *node,
     /*  */
1492                        bool optional)
1493 {
1494     /* If the cleanup is required, its orderings are optional, because they're
1495      * relevant only if both actions are required. Conversely, if the cleanup is
1496      * optional, the orderings make the then action required if the first action
1497      * becomes required.
1498      */
1499     uint32_t flag = optional? pcmk__ar_first_implies_then : pcmk__ar_ordered;
1500 
1501     CRM_CHECK((rsc != NULL) && (node != NULL), return);
1502 
1503     if (pcmk_is_set(rsc->flags, pcmk__rsc_failed)) {
1504         pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
1505                         rsc->id, pcmk__node_name(node));
1506         return;
1507     }
1508 
1509     if (node->details->unclean || !node->details->online) {
1510         pcmk__rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
1511                         rsc->id, pcmk__node_name(node));
1512         return;
1513     }
1514 
1515     crm_notice("Scheduling clean-up of %s on %s",
1516                rsc->id, pcmk__node_name(node));
1517     delete_action(rsc, node, optional);
1518 
1519     // stop -> clean-up -> start
1520     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
1521                                  rsc, PCMK_ACTION_DELETE, flag);
1522     pcmk__order_resource_actions(rsc, PCMK_ACTION_DELETE,
1523                                  rsc, PCMK_ACTION_START, flag);
1524 }
1525 
1526 /*!
1527  * \internal
1528  * \brief Add primitive meta-attributes relevant to graph actions to XML
1529  *
1530  * \param[in]     rsc  Primitive resource whose meta-attributes should be added
1531  * \param[in,out] xml  Transition graph action attributes XML to add to
1532  */
1533 void
1534 pcmk__primitive_add_graph_meta(const pcmk_resource_t *rsc, xmlNode *xml)
     /*  */
1535 {
1536     char *name = NULL;
1537     char *value = NULL;
1538     const pcmk_resource_t *parent = NULL;
1539 
1540     pcmk__assert(pcmk__is_primitive(rsc) && (xml != NULL));
1541 
1542     /* Clone instance numbers get set internally as meta-attributes, and are
1543      * needed in the transition graph (for example, to tell unique clone
1544      * instances apart).
1545      */
1546     value = g_hash_table_lookup(rsc->priv->meta, PCMK__META_CLONE);
1547     if (value != NULL) {
1548         name = crm_meta_name(PCMK__META_CLONE);
1549         crm_xml_add(xml, name, value);
1550         free(name);
1551     }
1552 
1553     // Not sure if this one is really needed ...
1554     value = g_hash_table_lookup(rsc->priv->meta, PCMK_META_REMOTE_NODE);
1555     if (value != NULL) {
1556         name = crm_meta_name(PCMK_META_REMOTE_NODE);
1557         crm_xml_add(xml, name, value);
1558         free(name);
1559     }
1560 
1561     /* The PCMK__META_CONTAINER meta-attribute can be set on the primitive
1562      * itself or one of its ancestors, so check them all and keep the highest.
1563      */
1564     for (parent = rsc; parent != NULL; parent = parent->priv->parent) {
1565         if (parent->priv->launcher != NULL) {
1566             crm_xml_add(xml, CRM_META "_" PCMK__META_CONTAINER,
1567                         parent->priv->launcher->id);
1568         }
1569     }
1570 
1571     /* Bundle replica children will get their external-ip set internally as a
1572      * meta-attribute. The graph action needs it, but under a different naming
1573      * convention than other meta-attributes.
1574      */
1575     value = g_hash_table_lookup(rsc->priv->meta, "external-ip");
1576     if (value != NULL) {
1577         crm_xml_add(xml, "pcmk_external_ip", value);
1578     }
1579 }
1580 
1581 // Primitive implementation of pcmk__assignment_methods_t:add_utilization()
1582 void
1583 pcmk__primitive_add_utilization(const pcmk_resource_t *rsc,
     /*  */
1584                                 const pcmk_resource_t *orig_rsc,
1585                                 GList *all_rscs, GHashTable *utilization)
1586 {
1587     pcmk__assert(pcmk__is_primitive(rsc) && (orig_rsc != NULL)
1588                  && (utilization != NULL));
1589 
1590     if (!pcmk_is_set(rsc->flags, pcmk__rsc_unassigned)) {
1591         return;
1592     }
1593 
1594     pcmk__rsc_trace(orig_rsc,
1595                     "%s: Adding primitive %s as colocated utilization",
1596                     orig_rsc->id, rsc->id);
1597     pcmk__release_node_capacity(utilization, rsc);
1598 }
1599 
1600 /*!
1601  * \internal
1602  * \brief Get epoch time of node's shutdown attribute (or now if none)
1603  *
1604  * \param[in,out] node  Node to check
1605  *
1606  * \return Epoch time corresponding to shutdown attribute if set or now if not
1607  */
1608 static time_t
1609 shutdown_time(pcmk_node_t *node)
     /*  */
1610 {
1611     const char *shutdown = pcmk__node_attr(node, PCMK__NODE_ATTR_SHUTDOWN, NULL,
1612                                            pcmk__rsc_node_current);
1613     time_t result = 0;
1614 
1615     if (shutdown != NULL) {
1616         long long result_ll;
1617         int rc = pcmk__scan_ll(shutdown, &result_ll, 0LL);
1618 
1619         if (rc == pcmk_rc_ok) {
1620             result = (time_t) result_ll;
1621         } else {
1622             crm_warn("Ignoring invalid value '%s' for %s "
1623                      PCMK__NODE_ATTR_SHUTDOWN " attribute: %s",
1624                      shutdown, pcmk__node_name(node), pcmk_rc_str(rc));
1625         }
1626     }
1627     if (result == 0) {
1628         result = pcmk__scheduler_epoch_time(node->priv->scheduler);
1629     }
1630     return result;
1631 }
1632 
1633 /*!
1634  * \internal
1635  * \brief Ban a resource from a node if it's not locked to the node
1636  *
1637  * \param[in]     data       Node to check
1638  * \param[in,out] user_data  Resource to check
1639  */
1640 static void
1641 ban_if_not_locked(gpointer data, gpointer user_data)
     /*  */
1642 {
1643     const pcmk_node_t *node = (const pcmk_node_t *) data;
1644     pcmk_resource_t *rsc = (pcmk_resource_t *) user_data;
1645 
1646     if (!pcmk__same_node(node, rsc->priv->lock_node)) {
1647         resource_location(rsc, node, -PCMK_SCORE_INFINITY,
1648                           PCMK_OPT_SHUTDOWN_LOCK, rsc->priv->scheduler);
1649     }
1650 }
1651 
1652 // Primitive implementation of pcmk__assignment_methods_t:shutdown_lock()
1653 void
1654 pcmk__primitive_shutdown_lock(pcmk_resource_t *rsc)
     /*  */
1655 {
1656     pcmk_scheduler_t *scheduler = NULL;
1657 
1658     pcmk__assert(pcmk__is_primitive(rsc));
1659     scheduler = rsc->priv->scheduler;
1660 
1661     // Fence devices and remote connections can't be locked
1662     if (pcmk_any_flags_set(rsc->flags, pcmk__rsc_fence_device
1663                                        |pcmk__rsc_is_remote_connection)) {
1664         return;
1665     }
1666 
1667     if (rsc->priv->lock_node != NULL) {
1668         // The lock was obtained from resource history
1669 
1670         if (rsc->priv->active_nodes != NULL) {
1671             /* The resource was started elsewhere even though it is now
1672              * considered locked. This shouldn't be possible, but as a
1673              * failsafe, we don't want to disturb the resource now.
1674              */
1675             pcmk__rsc_info(rsc,
1676                            "Cancelling shutdown lock "
1677                            "because %s is already active", rsc->id);
1678             pe__clear_resource_history(rsc, rsc->priv->lock_node);
1679             rsc->priv->lock_node = NULL;
1680             rsc->priv->lock_time = 0;
1681         }
1682 
1683     // Only a resource active on exactly one node can be locked
1684     } else if (pcmk__list_of_1(rsc->priv->active_nodes)) {
1685         pcmk_node_t *node = rsc->priv->active_nodes->data;
1686 
1687         if (node->details->shutdown) {
1688             if (node->details->unclean) {
1689                 pcmk__rsc_debug(rsc,
1690                                 "Not locking %s to unclean %s for shutdown",
1691                                 rsc->id, pcmk__node_name(node));
1692             } else {
1693                 rsc->priv->lock_node = node;
1694                 rsc->priv->lock_time = shutdown_time(node);
1695             }
1696         }
1697     }
1698 
1699     if (rsc->priv->lock_node == NULL) {
1700         // No lock needed
1701         return;
1702     }
1703 
1704     if (scheduler->priv->shutdown_lock_ms > 0U) {
1705         time_t lock_expiration = rsc->priv->lock_time
1706                                  + pcmk__timeout_ms2s(scheduler->priv->shutdown_lock_ms);
1707 
1708         pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
1709                        rsc->id, pcmk__node_name(rsc->priv->lock_node),
1710                        (long long) lock_expiration);
1711         pcmk__update_recheck_time(++lock_expiration, scheduler,
1712                                   "shutdown lock expiration");
1713     } else {
1714         pcmk__rsc_info(rsc, "Locking %s to %s due to shutdown",
1715                        rsc->id, pcmk__node_name(rsc->priv->lock_node));
1716     }
1717 
1718     // If resource is locked to one node, ban it from all other nodes
1719     g_list_foreach(scheduler->nodes, ban_if_not_locked, rsc);
1720 }
/* */
root/lib/pacemaker/pcmk_sched_primitive.c

DEFINITIONS
