lib/pacemaker/pcmk_sched

/* */
This source file includes following definitions.
sorted_allowed_nodes
assign_best_node
apply_this_with
apply_with_this
remote_connection_assigned
pcmk__primitive_assign
schedule_restart_actions
set_default_next_role
create_pending_start
schedule_role_transition_actions
pcmk__primitive_create_actions
rsc_avoids_remote_nodes
allowed_nodes_as_list
pcmk__primitive_internal_constraints
pcmk__primitive_apply_coloc_score
pcmk__primitive_action_flags
is_expected_node
stop_resource
start_resource
promote_resource
demote_resource
assert_role_error
pcmk__schedule_cleanup
pcmk__primitive_add_graph_meta
pcmk__primitive_add_utilization
shutdown_time
ban_if_not_locked
pcmk__primitive_shutdown_lock
   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdbool.h>
  13 
  14 #include <crm/msg_xml.h>
  15 #include <pacemaker-internal.h>
  16 
  17 #include "libpacemaker_private.h"
  18 
  19 static void stop_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
  20 static void start_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
  21 static void demote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional);
  22 static void promote_resource(pe_resource_t *rsc, pe_node_t *node,
  23                              bool optional);
  24 static void assert_role_error(pe_resource_t *rsc, pe_node_t *node,
  25                               bool optional);
  26 
  27 static enum rsc_role_e rsc_state_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
  28     /* This array lists the immediate next role when transitioning from one role
  29      * to a target role. For example, when going from Stopped to Promoted, the
  30      * next role is Unpromoted, because the resource must be started before it
  31      * can be promoted. The current state then becomes Started, which is fed
  32      * into this array again, giving a next role of Promoted.
  33      *
  34      * Current role       Immediate next role   Final target role
  35      * ------------       -------------------   -----------------
  36      */
  37     /* Unknown */       { RSC_ROLE_UNKNOWN,     /* Unknown */
  38                           RSC_ROLE_STOPPED,     /* Stopped */
  39                           RSC_ROLE_STOPPED,     /* Started */
  40                           RSC_ROLE_STOPPED,     /* Unpromoted */
  41                           RSC_ROLE_STOPPED,     /* Promoted */
  42                         },
  43     /* Stopped */       { RSC_ROLE_STOPPED,     /* Unknown */
  44                           RSC_ROLE_STOPPED,     /* Stopped */
  45                           RSC_ROLE_STARTED,     /* Started */
  46                           RSC_ROLE_UNPROMOTED,  /* Unpromoted */
  47                           RSC_ROLE_UNPROMOTED,  /* Promoted */
  48                         },
  49     /* Started */       { RSC_ROLE_STOPPED,     /* Unknown */
  50                           RSC_ROLE_STOPPED,     /* Stopped */
  51                           RSC_ROLE_STARTED,     /* Started */
  52                           RSC_ROLE_UNPROMOTED,  /* Unpromoted */
  53                           RSC_ROLE_PROMOTED,    /* Promoted */
  54                         },
  55     /* Unpromoted */    { RSC_ROLE_STOPPED,     /* Unknown */
  56                           RSC_ROLE_STOPPED,     /* Stopped */
  57                           RSC_ROLE_STOPPED,     /* Started */
  58                           RSC_ROLE_UNPROMOTED,  /* Unpromoted */
  59                           RSC_ROLE_PROMOTED,    /* Promoted */
  60                         },
  61     /* Promoted  */     { RSC_ROLE_STOPPED,     /* Unknown */
  62                           RSC_ROLE_UNPROMOTED,  /* Stopped */
  63                           RSC_ROLE_UNPROMOTED,  /* Started */
  64                           RSC_ROLE_UNPROMOTED,  /* Unpromoted */
  65                           RSC_ROLE_PROMOTED,    /* Promoted */
  66                         },
  67 };
  68 
  69 /*!
  70  * \internal
  71  * \brief Function to schedule actions needed for a role change
  72  *
  73  * \param[in,out] rsc       Resource whose role is changing
  74  * \param[in]     node      Node where resource will be in its next role
  75  * \param[in]     optional  Whether scheduled actions should be optional
  76  */
  77 typedef void (*rsc_transition_fn)(pe_resource_t *rsc, pe_node_t *node,
  78                                   bool optional);
  79 
  80 static rsc_transition_fn rsc_action_matrix[RSC_ROLE_MAX][RSC_ROLE_MAX] = {
  81     /* This array lists the function needed to transition directly from one role
  82      * to another. NULL indicates that nothing is needed.
  83      *
  84      * Current role         Transition function             Next role
  85      * ------------         -------------------             ----------
  86      */
  87     /* Unknown */       {   assert_role_error,              /* Unknown */
  88                             stop_resource,                  /* Stopped */
  89                             assert_role_error,              /* Started */
  90                             assert_role_error,              /* Unpromoted */
  91                             assert_role_error,              /* Promoted */
  92                         },
  93     /* Stopped */       {   assert_role_error,              /* Unknown */
  94                             NULL,                           /* Stopped */
  95                             start_resource,                 /* Started */
  96                             start_resource,                 /* Unpromoted */
  97                             assert_role_error,              /* Promoted */
  98                         },
  99     /* Started */       {   assert_role_error,              /* Unknown */
 100                             stop_resource,                  /* Stopped */
 101                             NULL,                           /* Started */
 102                             NULL,                           /* Unpromoted */
 103                             promote_resource,               /* Promoted */
 104                         },
 105     /* Unpromoted */    {   assert_role_error,              /* Unknown */
 106                             stop_resource,                  /* Stopped */
 107                             stop_resource,                  /* Started */
 108                             NULL,                           /* Unpromoted */
 109                             promote_resource,               /* Promoted */
 110                         },
 111     /* Promoted  */     {   assert_role_error,              /* Unknown */
 112                             demote_resource,                /* Stopped */
 113                             demote_resource,                /* Started */
 114                             demote_resource,                /* Unpromoted */
 115                             NULL,                           /* Promoted */
 116                         },
 117 };
 118 
 119 /*!
 120  * \internal
 121  * \brief Get a list of a resource's allowed nodes sorted by node weight
 122  *
 123  * \param[in] rsc  Resource to check
 124  *
 125  * \return List of allowed nodes sorted by node weight
 126  */
 127 static GList *
 128 sorted_allowed_nodes(const pe_resource_t *rsc)
     /*  */
 129 {
 130     if (rsc->allowed_nodes != NULL) {
 131         GList *nodes = g_hash_table_get_values(rsc->allowed_nodes);
 132 
 133         if (nodes != NULL) {
 134             return pcmk__sort_nodes(nodes, pe__current_node(rsc));
 135         }
 136     }
 137     return NULL;
 138 }
 139 
 140 /*!
 141  * \internal
 142  * \brief Assign a resource to its best allowed node, if possible
 143  *
 144  * \param[in,out] rsc     Resource to choose a node for
 145  * \param[in]     prefer  If not NULL, prefer this node when all else equal
 146  *
 147  * \return true if \p rsc could be assigned to a node, otherwise false
 148  */
 149 static bool
 150 assign_best_node(pe_resource_t *rsc, const pe_node_t *prefer)
     /*  */
 151 {
 152     GList *nodes = NULL;
 153     pe_node_t *chosen = NULL;
 154     pe_node_t *best = NULL;
 155     bool result = false;
 156     const pe_node_t *most_free_node = pcmk__ban_insufficient_capacity(rsc);
 157 
 158     if (prefer == NULL) {
 159         prefer = most_free_node;
 160     }
 161 
 162     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
 163         // We've already finished assignment of resources to nodes
 164         return rsc->allocated_to != NULL;
 165     }
 166 
 167     // Sort allowed nodes by weight
 168     nodes = sorted_allowed_nodes(rsc);
 169     if (nodes != NULL) {
 170         best = (pe_node_t *) nodes->data; // First node has best score
 171     }
 172 
 173     if ((prefer != NULL) && (nodes != NULL)) {
 174         // Get the allowed node version of prefer
 175         chosen = g_hash_table_lookup(rsc->allowed_nodes, prefer->details->id);
 176 
 177         if (chosen == NULL) {
 178             pe_rsc_trace(rsc, "Preferred node %s for %s was unknown",
 179                          pe__node_name(prefer), rsc->id);
 180 
 181         /* Favor the preferred node as long as its weight is at least as good as
 182          * the best allowed node's.
 183          *
 184          * An alternative would be to favor the preferred node even if the best
 185          * node is better, when the best node's weight is less than INFINITY.
 186          */
 187         } else if (chosen->weight < best->weight) {
 188             pe_rsc_trace(rsc, "Preferred node %s for %s was unsuitable",
 189                          pe__node_name(chosen), rsc->id);
 190             chosen = NULL;
 191 
 192         } else if (!pcmk__node_available(chosen, true, false)) {
 193             pe_rsc_trace(rsc, "Preferred node %s for %s was unavailable",
 194                          pe__node_name(chosen), rsc->id);
 195             chosen = NULL;
 196 
 197         } else {
 198             pe_rsc_trace(rsc,
 199                          "Chose preferred node %s for %s (ignoring %d candidates)",
 200                          pe__node_name(chosen), rsc->id, g_list_length(nodes));
 201         }
 202     }
 203 
 204     if ((chosen == NULL) && (best != NULL)) {
 205         /* Either there is no preferred node, or the preferred node is not
 206          * suitable, but another node is allowed to run the resource.
 207          */
 208 
 209         chosen = best;
 210 
 211         if (!pe_rsc_is_unique_clone(rsc->parent)
 212             && (chosen->weight > 0) // Zero not acceptable
 213             && pcmk__node_available(chosen, false, false)) {
 214             /* If the resource is already running on a node, prefer that node if
 215              * it is just as good as the chosen node.
 216              *
 217              * We don't do this for unique clone instances, because
 218              * distribute_children() has already assigned instances to their
 219              * running nodes when appropriate, and if we get here, we don't want
 220              * remaining unassigned instances to prefer a node that's already
 221              * running another instance.
 222              */
 223             pe_node_t *running = pe__current_node(rsc);
 224 
 225             if (running == NULL) {
 226                 // Nothing to do
 227 
 228             } else if (!pcmk__node_available(running, true, false)) {
 229                 pe_rsc_trace(rsc, "Current node for %s (%s) can't run resources",
 230                              rsc->id, pe__node_name(running));
 231 
 232             } else {
 233                 int nodes_with_best_score = 1;
 234 
 235                 for (GList *iter = nodes->next; iter; iter = iter->next) {
 236                     pe_node_t *allowed = (pe_node_t *) iter->data;
 237 
 238                     if (allowed->weight != chosen->weight) {
 239                         // The nodes are sorted by weight, so no more are equal
 240                         break;
 241                     }
 242                     if (pe__same_node(allowed, running)) {
 243                         // Scores are equal, so prefer the current node
 244                         chosen = allowed;
 245                     }
 246                     nodes_with_best_score++;
 247                 }
 248 
 249                 if (nodes_with_best_score > 1) {
 250                     do_crm_log(((chosen->weight >= INFINITY)? LOG_WARNING : LOG_INFO),
 251                                "Chose %s for %s from %d nodes with score %s",
 252                                pe__node_name(chosen), rsc->id,
 253                                nodes_with_best_score,
 254                                pcmk_readable_score(chosen->weight));
 255                 }
 256             }
 257         }
 258 
 259         pe_rsc_trace(rsc, "Chose %s for %s from %d candidates",
 260                      pe__node_name(chosen), rsc->id, g_list_length(nodes));
 261     }
 262 
 263     result = pcmk__finalize_assignment(rsc, chosen, false);
 264     g_list_free(nodes);
 265     return result;
 266 }
 267 
 268 /*!
 269  * \internal
 270  * \brief Apply a "this with" colocation to a node's allowed node scores
 271  *
 272  * \param[in,out] data       Colocation to apply
 273  * \param[in,out] user_data  Resource being assigned
 274  */
 275 static void
 276 apply_this_with(gpointer data, gpointer user_data)
     /*  */
 277 {
 278     pcmk__colocation_t *colocation = (pcmk__colocation_t *) data;
 279     pe_resource_t *rsc = (pe_resource_t *) user_data;
 280 
 281     GHashTable *archive = NULL;
 282     pe_resource_t *other = colocation->primary;
 283 
 284     // In certain cases, we will need to revert the node scores
 285     if ((colocation->dependent_role >= RSC_ROLE_PROMOTED)
 286         || ((colocation->score < 0) && (colocation->score > -INFINITY))) {
 287         archive = pcmk__copy_node_table(rsc->allowed_nodes);
 288     }
 289 
 290     pe_rsc_trace(rsc,
 291                  "%s: Assigning colocation %s primary %s first"
 292                  "(score=%d role=%s)",
 293                  rsc->id, colocation->id, other->id,
 294                  colocation->score, role2text(colocation->dependent_role));
 295     other->cmds->assign(other, NULL);
 296 
 297     // Apply the colocation score to this resource's allowed node scores
 298     rsc->cmds->apply_coloc_score(rsc, other, colocation, true);
 299     if ((archive != NULL)
 300         && !pcmk__any_node_available(rsc->allowed_nodes)) {
 301         pe_rsc_info(rsc,
 302                     "%s: Reverting scores from colocation with %s "
 303                     "because no nodes allowed",
 304                     rsc->id, other->id);
 305         g_hash_table_destroy(rsc->allowed_nodes);
 306         rsc->allowed_nodes = archive;
 307         archive = NULL;
 308     }
 309     if (archive != NULL) {
 310         g_hash_table_destroy(archive);
 311     }
 312 }
 313 
 314 /*!
 315  * \internal
 316  * \brief Apply a "with this" colocation to a node's allowed node scores
 317  *
 318  * \param[in,out] data       Colocation to apply
 319  * \param[in,out] user_data  Resource being assigned
 320  */
 321 static void
 322 apply_with_this(void *data, void *user_data)
     /*  */
 323 {
 324     pcmk__colocation_t *colocation = (pcmk__colocation_t *) data;
 325     pe_resource_t *rsc = (pe_resource_t *) user_data;
 326 
 327     pe_resource_t *other = colocation->dependent;
 328     const float factor = colocation->score / (float) INFINITY;
 329 
 330     if (!pcmk__colocation_has_influence(colocation, NULL)) {
 331         return;
 332     }
 333     pe_rsc_trace(rsc,
 334                  "%s: Incorporating attenuated %s assignment scores due "
 335                  "to colocation %s", rsc->id, other->id, colocation->id);
 336     pcmk__add_colocated_node_scores(other, rsc->id, &rsc->allowed_nodes,
 337                                     colocation->node_attribute, factor,
 338                                     pcmk__coloc_select_active);
 339 }
 340 
 341 /*!
 342  * \internal
 343  * \brief Update a Pacemaker Remote node once its connection has been assigned
 344  *
 345  * \param[in] connection  Connection resource that has been assigned
 346  */
 347 static void
 348 remote_connection_assigned(const pe_resource_t *connection)
     /*  */
 349 {
 350     pe_node_t *remote_node = pe_find_node(connection->cluster->nodes,
 351                                           connection->id);
 352 
 353     CRM_CHECK(remote_node != NULL, return);
 354 
 355     if ((connection->allocated_to != NULL)
 356         && (connection->next_role != RSC_ROLE_STOPPED)) {
 357 
 358         crm_trace("Pacemaker Remote node %s will be online",
 359                   remote_node->details->id);
 360         remote_node->details->online = TRUE;
 361         if (remote_node->details->unseen) {
 362             // Avoid unnecessary fence, since we will attempt connection
 363             remote_node->details->unclean = FALSE;
 364         }
 365 
 366     } else {
 367         crm_trace("Pacemaker Remote node %s will be shut down "
 368                   "(%sassigned connection's next role is %s)",
 369                   remote_node->details->id,
 370                   ((connection->allocated_to == NULL)? "un" : ""),
 371                   role2text(connection->next_role));
 372         remote_node->details->shutdown = TRUE;
 373     }
 374 }
 375 
 376 /*!
 377  * \internal
 378  * \brief Assign a primitive resource to a node
 379  *
 380  * \param[in,out] rsc     Resource to assign to a node
 381  * \param[in]     prefer  Node to prefer, if all else is equal
 382  *
 383  * \return Node that \p rsc is assigned to, if assigned entirely to one node
 384  */
 385 pe_node_t *
 386 pcmk__primitive_assign(pe_resource_t *rsc, const pe_node_t *prefer)
     /*  */
 387 {
 388     CRM_ASSERT(rsc != NULL);
 389 
 390     // Never assign a child without parent being assigned first
 391     if ((rsc->parent != NULL)
 392         && !pcmk_is_set(rsc->parent->flags, pe_rsc_allocating)) {
 393         pe_rsc_debug(rsc, "%s: Assigning parent %s first",
 394                      rsc->id, rsc->parent->id);
 395         rsc->parent->cmds->assign(rsc->parent, prefer);
 396     }
 397 
 398     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
 399         return rsc->allocated_to; // Assignment has already been done
 400     }
 401 
 402     // Ensure we detect assignment loops
 403     if (pcmk_is_set(rsc->flags, pe_rsc_allocating)) {
 404         pe_rsc_debug(rsc, "Breaking assignment loop involving %s", rsc->id);
 405         return NULL;
 406     }
 407     pe__set_resource_flags(rsc, pe_rsc_allocating);
 408 
 409     pe__show_node_weights(true, rsc, "Pre-assignment", rsc->allowed_nodes,
 410                           rsc->cluster);
 411 
 412     g_list_foreach(rsc->rsc_cons, apply_this_with, rsc);
 413     pe__show_node_weights(true, rsc, "Post-this-with", rsc->allowed_nodes,
 414                           rsc->cluster);
 415 
 416     g_list_foreach(rsc->rsc_cons_lhs, apply_with_this, rsc);
 417 
 418     if (rsc->next_role == RSC_ROLE_STOPPED) {
 419         pe_rsc_trace(rsc,
 420                      "Banning %s from all nodes because it will be stopped",
 421                      rsc->id);
 422         resource_location(rsc, NULL, -INFINITY, XML_RSC_ATTR_TARGET_ROLE,
 423                           rsc->cluster);
 424 
 425     } else if ((rsc->next_role > rsc->role)
 426                && !pcmk_is_set(rsc->cluster->flags, pe_flag_have_quorum)
 427                && (rsc->cluster->no_quorum_policy == no_quorum_freeze)) {
 428         crm_notice("Resource %s cannot be elevated from %s to %s due to "
 429                    "no-quorum-policy=freeze",
 430                    rsc->id, role2text(rsc->role), role2text(rsc->next_role));
 431         pe__set_next_role(rsc, rsc->role, "no-quorum-policy=freeze");
 432     }
 433 
 434     pe__show_node_weights(!pcmk_is_set(rsc->cluster->flags, pe_flag_show_scores),
 435                           rsc, __func__, rsc->allowed_nodes, rsc->cluster);
 436 
 437     // Unmanage resource if fencing is enabled but no device is configured
 438     if (pcmk_is_set(rsc->cluster->flags, pe_flag_stonith_enabled)
 439         && !pcmk_is_set(rsc->cluster->flags, pe_flag_have_stonith_resource)) {
 440         pe__clear_resource_flags(rsc, pe_rsc_managed);
 441     }
 442 
 443     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
 444         // Unmanaged resources stay on their current node
 445         const char *reason = NULL;
 446         pe_node_t *assign_to = NULL;
 447 
 448         pe__set_next_role(rsc, rsc->role, "unmanaged");
 449         assign_to = pe__current_node(rsc);
 450         if (assign_to == NULL) {
 451             reason = "inactive";
 452         } else if (rsc->role == RSC_ROLE_PROMOTED) {
 453             reason = "promoted";
 454         } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
 455             reason = "failed";
 456         } else {
 457             reason = "active";
 458         }
 459         pe_rsc_info(rsc, "Unmanaged resource %s assigned to %s: %s", rsc->id,
 460                     (assign_to? assign_to->details->uname : "no node"), reason);
 461         pcmk__finalize_assignment(rsc, assign_to, true);
 462 
 463     } else if (pcmk_is_set(rsc->cluster->flags, pe_flag_stop_everything)) {
 464         pe_rsc_debug(rsc, "Forcing %s to stop: stop-all-resources", rsc->id);
 465         pcmk__finalize_assignment(rsc, NULL, true);
 466 
 467     } else if (pcmk_is_set(rsc->flags, pe_rsc_provisional)
 468                && assign_best_node(rsc, prefer)) {
 469         // Assignment successful
 470 
 471     } else if (rsc->allocated_to == NULL) {
 472         if (!pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
 473             pe_rsc_info(rsc, "Resource %s cannot run anywhere", rsc->id);
 474         } else if (rsc->running_on != NULL) {
 475             pe_rsc_info(rsc, "Stopping orphan resource %s", rsc->id);
 476         }
 477 
 478     } else {
 479         pe_rsc_debug(rsc, "%s: pre-assigned to %s", rsc->id,
 480                      pe__node_name(rsc->allocated_to));
 481     }
 482 
 483     pe__clear_resource_flags(rsc, pe_rsc_allocating);
 484 
 485     if (rsc->is_remote_node) {
 486         remote_connection_assigned(rsc);
 487     }
 488 
 489     return rsc->allocated_to;
 490 }
 491 
 492 /*!
 493  * \internal
 494  * \brief Schedule actions to bring resource down and back to current role
 495  *
 496  * \param[in,out] rsc           Resource to restart
 497  * \param[in]     current       Node that resource should be brought down on
 498  * \param[in]     need_stop     Whether the resource must be stopped
 499  * \param[in]     need_promote  Whether the resource must be promoted
 500  *
 501  * \return Role that resource would have after scheduled actions are taken
 502  */
 503 static void
 504 schedule_restart_actions(pe_resource_t *rsc, pe_node_t *current,
     /*  */
 505                          bool need_stop, bool need_promote)
 506 {
 507     enum rsc_role_e role = rsc->role;
 508     enum rsc_role_e next_role;
 509     rsc_transition_fn fn = NULL;
 510 
 511     pe__set_resource_flags(rsc, pe_rsc_restarting);
 512 
 513     // Bring resource down to a stop on its current node
 514     while (role != RSC_ROLE_STOPPED) {
 515         next_role = rsc_state_matrix[role][RSC_ROLE_STOPPED];
 516         pe_rsc_trace(rsc, "Creating %s action to take %s down from %s to %s",
 517                      (need_stop? "required" : "optional"), rsc->id,
 518                      role2text(role), role2text(next_role));
 519         fn = rsc_action_matrix[role][next_role];
 520         if (fn == NULL) {
 521             break;
 522         }
 523         fn(rsc, current, !need_stop);
 524         role = next_role;
 525     }
 526 
 527     // Bring resource up to its next role on its next node
 528     while ((rsc->role <= rsc->next_role) && (role != rsc->role)
 529            && !pcmk_is_set(rsc->flags, pe_rsc_block)) {
 530         bool required = need_stop;
 531 
 532         next_role = rsc_state_matrix[role][rsc->role];
 533         if ((next_role == RSC_ROLE_PROMOTED) && need_promote) {
 534             required = true;
 535         }
 536         pe_rsc_trace(rsc, "Creating %s action to take %s up from %s to %s",
 537                      (required? "required" : "optional"), rsc->id,
 538                      role2text(role), role2text(next_role));
 539         fn = rsc_action_matrix[role][next_role];
 540         if (fn == NULL) {
 541             break;
 542         }
 543         fn(rsc, rsc->allocated_to, !required);
 544         role = next_role;
 545     }
 546 
 547     pe__clear_resource_flags(rsc, pe_rsc_restarting);
 548 }
 549 
 550 /*!
 551  * \internal
 552  * \brief If a resource's next role is not explicitly specified, set a default
 553  *
 554  * \param[in,out] rsc  Resource to set next role for
 555  *
 556  * \return "explicit" if next role was explicitly set, otherwise "implicit"
 557  */
 558 static const char *
 559 set_default_next_role(pe_resource_t *rsc)
     /*  */
 560 {
 561     if (rsc->next_role != RSC_ROLE_UNKNOWN) {
 562         return "explicit";
 563     }
 564 
 565     if (rsc->allocated_to == NULL) {
 566         pe__set_next_role(rsc, RSC_ROLE_STOPPED, "assignment");
 567     } else {
 568         pe__set_next_role(rsc, RSC_ROLE_STARTED, "assignment");
 569     }
 570     return "implicit";
 571 }
 572 
 573 /*!
 574  * \internal
 575  * \brief Create an action to represent an already pending start
 576  *
 577  * \param[in,out] rsc  Resource to create start action for
 578  */
 579 static void
 580 create_pending_start(pe_resource_t *rsc)
     /*  */
 581 {
 582     pe_action_t *start = NULL;
 583 
 584     pe_rsc_trace(rsc,
 585                  "Creating action for %s to represent already pending start",
 586                  rsc->id);
 587     start = start_action(rsc, rsc->allocated_to, TRUE);
 588     pe__set_action_flags(start, pe_action_print_always);
 589 }
 590 
 591 /*!
 592  * \internal
 593  * \brief Schedule actions needed to take a resource to its next role
 594  *
 595  * \param[in,out] rsc  Resource to schedule actions for
 596  */
 597 static void
 598 schedule_role_transition_actions(pe_resource_t *rsc)
     /*  */
 599 {
 600     enum rsc_role_e role = rsc->role;
 601 
 602     while (role != rsc->next_role) {
 603         enum rsc_role_e next_role = rsc_state_matrix[role][rsc->next_role];
 604         rsc_transition_fn fn = NULL;
 605 
 606         pe_rsc_trace(rsc,
 607                      "Creating action to take %s from %s to %s (ending at %s)",
 608                      rsc->id, role2text(role), role2text(next_role),
 609                      role2text(rsc->next_role));
 610         fn = rsc_action_matrix[role][next_role];
 611         if (fn == NULL) {
 612             break;
 613         }
 614         fn(rsc, rsc->allocated_to, false);
 615         role = next_role;
 616     }
 617 }
 618 
 619 /*!
 620  * \internal
 621  * \brief Create all actions needed for a given primitive resource
 622  *
 623  * \param[in,out] rsc  Primitive resource to create actions for
 624  */
 625 void
 626 pcmk__primitive_create_actions(pe_resource_t *rsc)
     /*  */
 627 {
 628     bool need_stop = false;
 629     bool need_promote = false;
 630     bool is_moving = false;
 631     bool allow_migrate = false;
 632     bool multiply_active = false;
 633 
 634     pe_node_t *current = NULL;
 635     unsigned int num_all_active = 0;
 636     unsigned int num_clean_active = 0;
 637     const char *next_role_source = NULL;
 638 
 639     CRM_ASSERT(rsc != NULL);
 640 
 641     next_role_source = set_default_next_role(rsc);
 642     pe_rsc_trace(rsc,
 643                  "Creating all actions for %s transition from %s to %s "
 644                  "(%s) on %s",
 645                  rsc->id, role2text(rsc->role), role2text(rsc->next_role),
 646                  next_role_source, pe__node_name(rsc->allocated_to));
 647 
 648     current = pe__find_active_on(rsc, &num_all_active, &num_clean_active);
 649 
 650     g_list_foreach(rsc->dangling_migrations, pcmk__abort_dangling_migration,
 651                    rsc);
 652 
 653     if ((current != NULL) && (rsc->allocated_to != NULL)
 654         && (current->details != rsc->allocated_to->details)
 655         && (rsc->next_role >= RSC_ROLE_STARTED)) {
 656 
 657         pe_rsc_trace(rsc, "Moving %s from %s to %s",
 658                      rsc->id, pe__node_name(current),
 659                      pe__node_name(rsc->allocated_to));
 660         is_moving = true;
 661         allow_migrate = pcmk__rsc_can_migrate(rsc, current);
 662 
 663         // This is needed even if migrating (though I'm not sure why ...)
 664         need_stop = true;
 665     }
 666 
 667     // Check whether resource is partially migrated and/or multiply active
 668     if ((rsc->partial_migration_source != NULL)
 669         && (rsc->partial_migration_target != NULL)
 670         && allow_migrate && (num_all_active == 2)
 671         && pe__same_node(current, rsc->partial_migration_source)
 672         && pe__same_node(rsc->allocated_to, rsc->partial_migration_target)) {
 673         /* A partial migration is in progress, and the migration target remains
 674          * the same as when the migration began.
 675          */
 676         pe_rsc_trace(rsc, "Partial migration of %s from %s to %s will continue",
 677                      rsc->id, pe__node_name(rsc->partial_migration_source),
 678                      pe__node_name(rsc->partial_migration_target));
 679 
 680     } else if ((rsc->partial_migration_source != NULL)
 681                || (rsc->partial_migration_target != NULL)) {
 682         // A partial migration is in progress but can't be continued
 683 
 684         if (num_all_active > 2) {
 685             // The resource is migrating *and* multiply active!
 686             crm_notice("Forcing recovery of %s because it is migrating "
 687                        "from %s to %s and possibly active elsewhere",
 688                        rsc->id, pe__node_name(rsc->partial_migration_source),
 689                        pe__node_name(rsc->partial_migration_target));
 690         } else {
 691             // The migration source or target isn't available
 692             crm_notice("Forcing recovery of %s because it can no longer "
 693                        "migrate from %s to %s",
 694                        rsc->id, pe__node_name(rsc->partial_migration_source),
 695                        pe__node_name(rsc->partial_migration_target));
 696         }
 697         need_stop = true;
 698         rsc->partial_migration_source = rsc->partial_migration_target = NULL;
 699         allow_migrate = false;
 700 
 701     } else if (pcmk_is_set(rsc->flags, pe_rsc_needs_fencing)) {
 702         multiply_active = (num_all_active > 1);
 703     } else {
 704         /* If a resource has "requires" set to nothing or quorum, don't consider
 705          * it active on unclean nodes (similar to how all resources behave when
 706          * stonith-enabled is false). We can start such resources elsewhere
 707          * before fencing completes, and if we considered the resource active on
 708          * the failed node, we would attempt recovery for being active on
 709          * multiple nodes.
 710          */
 711         multiply_active = (num_clean_active > 1);
 712     }
 713 
 714     if (multiply_active) {
 715         const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 716 
 717         // Resource was (possibly) incorrectly multiply active
 718         pe_proc_err("%s resource %s might be active on %u nodes (%s)",
 719                     pcmk__s(class, "Untyped"), rsc->id, num_all_active,
 720                     recovery2text(rsc->recovery_type));
 721         crm_notice("See https://wiki.clusterlabs.org/wiki/FAQ"
 722                    "#Resource_is_Too_Active for more information");
 723 
 724         switch (rsc->recovery_type) {
 725             case recovery_stop_start:
 726                 need_stop = true;
 727                 break;
 728             case recovery_stop_unexpected:
 729                 need_stop = true; // stop_resource() will skip expected node
 730                 pe__set_resource_flags(rsc, pe_rsc_stop_unexpected);
 731                 break;
 732             default:
 733                 break;
 734         }
 735 
 736     } else {
 737         pe__clear_resource_flags(rsc, pe_rsc_stop_unexpected);
 738     }
 739 
 740     if (pcmk_is_set(rsc->flags, pe_rsc_start_pending)) {
 741         create_pending_start(rsc);
 742     }
 743 
 744     if (is_moving) {
 745         // Remaining tests are only for resources staying where they are
 746 
 747     } else if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
 748         if (pcmk_is_set(rsc->flags, pe_rsc_stop)) {
 749             need_stop = true;
 750             pe_rsc_trace(rsc, "Recovering %s", rsc->id);
 751         } else {
 752             pe_rsc_trace(rsc, "Recovering %s by demotion", rsc->id);
 753             if (rsc->next_role == RSC_ROLE_PROMOTED) {
 754                 need_promote = true;
 755             }
 756         }
 757 
 758     } else if (pcmk_is_set(rsc->flags, pe_rsc_block)) {
 759         pe_rsc_trace(rsc, "Blocking further actions on %s", rsc->id);
 760         need_stop = true;
 761 
 762     } else if ((rsc->role > RSC_ROLE_STARTED) && (current != NULL)
 763                && (rsc->allocated_to != NULL)) {
 764         pe_action_t *start = NULL;
 765 
 766         pe_rsc_trace(rsc, "Creating start action for promoted resource %s",
 767                      rsc->id);
 768         start = start_action(rsc, rsc->allocated_to, TRUE);
 769         if (!pcmk_is_set(start->flags, pe_action_optional)) {
 770             // Recovery of a promoted resource
 771             pe_rsc_trace(rsc, "%s restart is required for recovery", rsc->id);
 772             need_stop = true;
 773         }
 774     }
 775 
 776     // Create any actions needed to bring resource down and back up to same role
 777     schedule_restart_actions(rsc, current, need_stop, need_promote);
 778 
 779     // Create any actions needed to take resource from this role to the next
 780     schedule_role_transition_actions(rsc);
 781 
 782     pcmk__create_recurring_actions(rsc);
 783 
 784     if (allow_migrate) {
 785         pcmk__create_migration_actions(rsc, current);
 786     }
 787 }
 788 
 789 /*!
 790  * \internal
 791  * \brief Ban a resource from any allowed nodes that are Pacemaker Remote nodes
 792  *
 793  * \param[in] rsc  Resource to check
 794  */
 795 static void
 796 rsc_avoids_remote_nodes(const pe_resource_t *rsc)
     /*  */
 797 {
 798     GHashTableIter iter;
 799     pe_node_t *node = NULL;
 800 
 801     g_hash_table_iter_init(&iter, rsc->allowed_nodes);
 802     while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
 803         if (node->details->remote_rsc != NULL) {
 804             node->weight = -INFINITY;
 805         }
 806     }
 807 }
 808 
 809 /*!
 810  * \internal
 811  * \brief Return allowed nodes as (possibly sorted) list
 812  *
 813  * Convert a resource's hash table of allowed nodes to a list. If printing to
 814  * stdout, sort the list, to keep action ID numbers consistent for regression
 815  * test output (while avoiding the performance hit on a live cluster).
 816  *
 817  * \param[in] rsc       Resource to check for allowed nodes
 818  *
 819  * \return List of resource's allowed nodes
 820  * \note Callers should take care not to rely on the list being sorted.
 821  */
 822 static GList *
 823 allowed_nodes_as_list(const pe_resource_t *rsc)
     /*  */
 824 {
 825     GList *allowed_nodes = NULL;
 826 
 827     if (rsc->allowed_nodes) {
 828         allowed_nodes = g_hash_table_get_values(rsc->allowed_nodes);
 829     }
 830 
 831     if (!pcmk__is_daemon) {
 832         allowed_nodes = g_list_sort(allowed_nodes, pe__cmp_node_name);
 833     }
 834 
 835     return allowed_nodes;
 836 }
 837 
 838 /*!
 839  * \internal
 840  * \brief Create implicit constraints needed for a primitive resource
 841  *
 842  * \param[in,out] rsc  Primitive resource to create implicit constraints for
 843  */
 844 void
 845 pcmk__primitive_internal_constraints(pe_resource_t *rsc)
     /*  */
 846 {
 847     pe_resource_t *top = NULL;
 848     GList *allowed_nodes = NULL;
 849     bool check_unfencing = false;
 850     bool check_utilization = false;
 851 
 852     CRM_ASSERT(rsc != NULL);
 853 
 854     if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
 855         pe_rsc_trace(rsc,
 856                      "Skipping implicit constraints for unmanaged resource %s",
 857                      rsc->id);
 858         return;
 859     }
 860 
 861     top = uber_parent(rsc);
 862 
 863     // Whether resource requires unfencing
 864     check_unfencing = !pcmk_is_set(rsc->flags, pe_rsc_fence_device)
 865                       && pcmk_is_set(rsc->cluster->flags, pe_flag_enable_unfencing)
 866                       && pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing);
 867 
 868     // Whether a non-default placement strategy is used
 869     check_utilization = (g_hash_table_size(rsc->utilization) > 0)
 870                          && !pcmk__str_eq(rsc->cluster->placement_strategy,
 871                                           "default", pcmk__str_casei);
 872 
 873     // Order stops before starts (i.e. restart)
 874     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
 875                        rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
 876                        pe_order_optional|pe_order_implies_then|pe_order_restart,
 877                        rsc->cluster);
 878 
 879     // Promotable ordering: demote before stop, start before promote
 880     if (pcmk_is_set(top->flags, pe_rsc_promotable)
 881         || (rsc->role > RSC_ROLE_UNPROMOTED)) {
 882 
 883         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_DEMOTE, 0), NULL,
 884                            rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
 885                            pe_order_promoted_implies_first, rsc->cluster);
 886 
 887         pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_START, 0), NULL,
 888                            rsc, pcmk__op_key(rsc->id, RSC_PROMOTE, 0), NULL,
 889                            pe_order_runnable_left, rsc->cluster);
 890     }
 891 
 892     // Don't clear resource history if probing on same node
 893     pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, CRM_OP_LRM_DELETE, 0),
 894                        NULL, rsc, pcmk__op_key(rsc->id, RSC_STATUS, 0),
 895                        NULL, pe_order_same_node|pe_order_then_cancels_first,
 896                        rsc->cluster);
 897 
 898     // Certain checks need allowed nodes
 899     if (check_unfencing || check_utilization || (rsc->container != NULL)) {
 900         allowed_nodes = allowed_nodes_as_list(rsc);
 901     }
 902 
 903     if (check_unfencing) {
 904         g_list_foreach(allowed_nodes, pcmk__order_restart_vs_unfence, rsc);
 905     }
 906 
 907     if (check_utilization) {
 908         pcmk__create_utilization_constraints(rsc, allowed_nodes);
 909     }
 910 
 911     if (rsc->container != NULL) {
 912         pe_resource_t *remote_rsc = NULL;
 913 
 914         if (rsc->is_remote_node) {
 915             // rsc is the implicit remote connection for a guest or bundle node
 916 
 917             /* Guest resources are not allowed to run on Pacemaker Remote nodes,
 918              * to avoid nesting remotes. However, bundles are allowed.
 919              */
 920             if (!pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
 921                 rsc_avoids_remote_nodes(rsc->container);
 922             }
 923 
 924             /* If someone cleans up a guest or bundle node's container, we will
 925              * likely schedule a (re-)probe of the container and recovery of the
 926              * connection. Order the connection stop after the container probe,
 927              * so that if we detect the container running, we will trigger a new
 928              * transition and avoid the unnecessary recovery.
 929              */
 930             pcmk__order_resource_actions(rsc->container, RSC_STATUS, rsc,
 931                                          RSC_STOP, pe_order_optional);
 932 
 933         /* A user can specify that a resource must start on a Pacemaker Remote
 934          * node by explicitly configuring it with the container=NODENAME
 935          * meta-attribute. This is of questionable merit, since location
 936          * constraints can accomplish the same thing. But we support it, so here
 937          * we check whether a resource (that is not itself a remote connection)
 938          * has container set to a remote node or guest node resource.
 939          */
 940         } else if (rsc->container->is_remote_node) {
 941             remote_rsc = rsc->container;
 942         } else  {
 943             remote_rsc = pe__resource_contains_guest_node(rsc->cluster,
 944                                                           rsc->container);
 945         }
 946 
 947         if (remote_rsc != NULL) {
 948             /* Force the resource on the Pacemaker Remote node instead of
 949              * colocating the resource with the container resource.
 950              */
 951             for (GList *item = allowed_nodes; item; item = item->next) {
 952                 pe_node_t *node = item->data;
 953 
 954                 if (node->details->remote_rsc != remote_rsc) {
 955                     node->weight = -INFINITY;
 956                 }
 957             }
 958 
 959         } else {
 960             /* This resource is either a filler for a container that does NOT
 961              * represent a Pacemaker Remote node, or a Pacemaker Remote
 962              * connection resource for a guest node or bundle.
 963              */
 964             int score;
 965 
 966             crm_trace("Order and colocate %s relative to its container %s",
 967                       rsc->id, rsc->container->id);
 968 
 969             pcmk__new_ordering(rsc->container,
 970                                pcmk__op_key(rsc->container->id, RSC_START, 0),
 971                                NULL, rsc, pcmk__op_key(rsc->id, RSC_START, 0),
 972                                NULL,
 973                                pe_order_implies_then|pe_order_runnable_left,
 974                                rsc->cluster);
 975 
 976             pcmk__new_ordering(rsc, pcmk__op_key(rsc->id, RSC_STOP, 0), NULL,
 977                                rsc->container,
 978                                pcmk__op_key(rsc->container->id, RSC_STOP, 0),
 979                                NULL, pe_order_implies_first, rsc->cluster);
 980 
 981             if (pcmk_is_set(rsc->flags, pe_rsc_allow_remote_remotes)) {
 982                 score = 10000;    /* Highly preferred but not essential */
 983             } else {
 984                 score = INFINITY; /* Force them to run on the same host */
 985             }
 986             pcmk__new_colocation("resource-with-container", NULL, score, rsc,
 987                                  rsc->container, NULL, NULL, true,
 988                                  rsc->cluster);
 989         }
 990     }
 991 
 992     if (rsc->is_remote_node || pcmk_is_set(rsc->flags, pe_rsc_fence_device)) {
 993         /* Remote connections and fencing devices are not allowed to run on
 994          * Pacemaker Remote nodes
 995          */
 996         rsc_avoids_remote_nodes(rsc);
 997     }
 998     g_list_free(allowed_nodes);
 999 }
1000 
1001 /*!
1002  * \internal
1003  * \brief Apply a colocation's score to node weights or resource priority
1004  *
1005  * Given a colocation constraint, apply its score to the dependent's
1006  * allowed node weights (if we are still placing resources) or priority (if
1007  * we are choosing promotable clone instance roles).
1008  *
1009  * \param[in,out] dependent      Dependent resource in colocation
1010  * \param[in]     primary        Primary resource in colocation
1011  * \param[in]     colocation     Colocation constraint to apply
1012  * \param[in] for_dependent  true if called on behalf of dependent
1013  */
1014 void
1015 pcmk__primitive_apply_coloc_score(pe_resource_t *dependent,
     /*  */
1016                                   const pe_resource_t *primary,
1017                                   const pcmk__colocation_t *colocation,
1018                                   bool for_dependent)
1019 {
1020     enum pcmk__coloc_affects filter_results;
1021 
1022     CRM_CHECK((colocation != NULL) && (dependent != NULL) && (primary != NULL),
1023               return);
1024 
1025     if (for_dependent) {
1026         // Always process on behalf of primary resource
1027         primary->cmds->apply_coloc_score(dependent, primary, colocation, false);
1028         return;
1029     }
1030 
1031     filter_results = pcmk__colocation_affects(dependent, primary, colocation,
1032                                               false);
1033     pe_rsc_trace(dependent, "%s %s with %s (%s, score=%d, filter=%d)",
1034                  ((colocation->score > 0)? "Colocating" : "Anti-colocating"),
1035                  dependent->id, primary->id, colocation->id, colocation->score,
1036                  filter_results);
1037 
1038     switch (filter_results) {
1039         case pcmk__coloc_affects_role:
1040             pcmk__apply_coloc_to_priority(dependent, primary, colocation);
1041             break;
1042         case pcmk__coloc_affects_location:
1043             pcmk__apply_coloc_to_weights(dependent, primary, colocation);
1044             break;
1045         default: // pcmk__coloc_affects_nothing
1046             return;
1047     }
1048 }
1049 
1050 /*!
1051  * \internal
1052  * \brief Return action flags for a given primitive resource action
1053  *
1054  * \param[in,out] action  Action to get flags for
1055  * \param[in]     node    If not NULL, limit effects to this node (ignored)
1056  *
1057  * \return Flags appropriate to \p action on \p node
1058  */
1059 enum pe_action_flags
1060 pcmk__primitive_action_flags(pe_action_t *action, const pe_node_t *node)
     /*  */
1061 {
1062     CRM_ASSERT(action != NULL);
1063     return action->flags;
1064 }
1065 
1066 /*!
1067  * \internal
1068  * \brief Check whether a node is a multiply active resource's expected node
1069  *
1070  * \param[in] rsc  Resource to check
1071  * \param[in] node  Node to check
1072  *
1073  * \return true if \p rsc is multiply active with multiple-active set to
1074  *         stop_unexpected, and \p node is the node where it will remain active
1075  * \note This assumes that the resource's next role cannot be changed to stopped
1076  *       after this is called, which should be reasonable if status has already
1077  *       been unpacked and resources have been assigned to nodes.
1078  */
1079 static bool
1080 is_expected_node(const pe_resource_t *rsc, const pe_node_t *node)
     /*  */
1081 {
1082     return pcmk_all_flags_set(rsc->flags,
1083                               pe_rsc_stop_unexpected|pe_rsc_restarting)
1084            && (rsc->next_role > RSC_ROLE_STOPPED)
1085            && pe__same_node(rsc->allocated_to, node);
1086 }
1087 
1088 /*!
1089  * \internal
1090  * \brief Schedule actions needed to stop a resource wherever it is active
1091  *
1092  * \param[in,out] rsc       Resource being stopped
1093  * \param[in]     node      Node where resource is being stopped (ignored)
1094  * \param[in]     optional  Whether actions should be optional
1095  */
1096 static void
1097 stop_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
     /*  */
1098 {
1099     for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
1100         pe_node_t *current = (pe_node_t *) iter->data;
1101         pe_action_t *stop = NULL;
1102 
1103         if (is_expected_node(rsc, current)) {
1104             /* We are scheduling restart actions for a multiply active resource
1105              * with multiple-active=stop_unexpected, and this is where it should
1106              * not be stopped.
1107              */
1108             pe_rsc_trace(rsc,
1109                          "Skipping stop of multiply active resource %s "
1110                          "on expected node %s",
1111                          rsc->id, pe__node_name(current));
1112             continue;
1113         }
1114 
1115         if (rsc->partial_migration_target != NULL) {
1116             // Continue migration if node originally was and remains target
1117             if (pe__same_node(current, rsc->partial_migration_target)
1118                 && pe__same_node(current, rsc->allocated_to)) {
1119                 pe_rsc_trace(rsc,
1120                              "Skipping stop of %s on %s "
1121                              "because partial migration there will continue",
1122                              rsc->id, pe__node_name(current));
1123                 continue;
1124             } else {
1125                 pe_rsc_trace(rsc,
1126                              "Forcing stop of %s on %s "
1127                              "because migration target changed",
1128                              rsc->id, pe__node_name(current));
1129                 optional = false;
1130             }
1131         }
1132 
1133         pe_rsc_trace(rsc, "Scheduling stop of %s on %s",
1134                      rsc->id, pe__node_name(current));
1135         stop = stop_action(rsc, current, optional);
1136 
1137         if (rsc->allocated_to == NULL) {
1138             pe_action_set_reason(stop, "node availability", true);
1139         } else if (pcmk_all_flags_set(rsc->flags, pe_rsc_restarting
1140                                                   |pe_rsc_stop_unexpected)) {
1141             /* We are stopping a multiply active resource on a node that is
1142              * not its expected node, and we are still scheduling restart
1143              * actions, so the stop is for being multiply active.
1144              */
1145             pe_action_set_reason(stop, "being multiply active", true);
1146         }
1147 
1148         if (!pcmk_is_set(rsc->flags, pe_rsc_managed)) {
1149             pe__clear_action_flags(stop, pe_action_runnable);
1150         }
1151 
1152         if (pcmk_is_set(rsc->cluster->flags, pe_flag_remove_after_stop)) {
1153             pcmk__schedule_cleanup(rsc, current, optional);
1154         }
1155 
1156         if (pcmk_is_set(rsc->flags, pe_rsc_needs_unfencing)) {
1157             pe_action_t *unfence = pe_fence_op(current, "on", true, NULL, false,
1158                                                rsc->cluster);
1159 
1160             order_actions(stop, unfence, pe_order_implies_first);
1161             if (!pcmk__node_unfenced(current)) {
1162                 pe_proc_err("Stopping %s until %s can be unfenced",
1163                             rsc->id, pe__node_name(current));
1164             }
1165         }
1166     }
1167 }
1168 
1169 /*!
1170  * \internal
1171  * \brief Schedule actions needed to start a resource on a node
1172  *
1173  * \param[in,out] rsc       Resource being started
1174  * \param[in]     node      Node where resource should be started
1175  * \param[in]     optional  Whether actions should be optional
1176  */
1177 static void
1178 start_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
     /*  */
1179 {
1180     pe_action_t *start = NULL;
1181 
1182     CRM_ASSERT(node != NULL);
1183 
1184     pe_rsc_trace(rsc, "Scheduling %s start of %s on %s (score %d)",
1185                  (optional? "optional" : "required"), rsc->id,
1186                  pe__node_name(node), node->weight);
1187     start = start_action(rsc, node, TRUE);
1188 
1189     pcmk__order_vs_unfence(rsc, node, start, pe_order_implies_then);
1190 
1191     if (pcmk_is_set(start->flags, pe_action_runnable) && !optional) {
1192         pe__clear_action_flags(start, pe_action_optional);
1193     }
1194 
1195     if (is_expected_node(rsc, node)) {
1196         /* This could be a problem if the start becomes necessary for other
1197          * reasons later.
1198          */
1199         pe_rsc_trace(rsc,
1200                      "Start of multiply active resouce %s "
1201                      "on expected node %s will be a pseudo-action",
1202                      rsc->id, pe__node_name(node));
1203         pe__set_action_flags(start, pe_action_pseudo);
1204     }
1205 }
1206 
1207 /*!
1208  * \internal
1209  * \brief Schedule actions needed to promote a resource on a node
1210  *
1211  * \param[in,out] rsc       Resource being promoted
1212  * \param[in]     node      Node where resource should be promoted
1213  * \param[in]     optional  Whether actions should be optional
1214  */
1215 static void
1216 promote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
     /*  */
1217 {
1218     GList *iter = NULL;
1219     GList *action_list = NULL;
1220     bool runnable = true;
1221 
1222     CRM_ASSERT(node != NULL);
1223 
1224     // Any start must be runnable for promotion to be runnable
1225     action_list = pe__resource_actions(rsc, node, RSC_START, true);
1226     for (iter = action_list; iter != NULL; iter = iter->next) {
1227         pe_action_t *start = (pe_action_t *) iter->data;
1228 
1229         if (!pcmk_is_set(start->flags, pe_action_runnable)) {
1230             runnable = false;
1231         }
1232     }
1233     g_list_free(action_list);
1234 
1235     if (runnable) {
1236         pe_action_t *promote = promote_action(rsc, node, optional);
1237 
1238         pe_rsc_trace(rsc, "Scheduling %s promotion of %s on %s",
1239                      (optional? "optional" : "required"), rsc->id,
1240                      pe__node_name(node));
1241 
1242         if (is_expected_node(rsc, node)) {
1243             /* This could be a problem if the promote becomes necessary for
1244              * other reasons later.
1245              */
1246             pe_rsc_trace(rsc,
1247                          "Promotion of multiply active resouce %s "
1248                          "on expected node %s will be a pseudo-action",
1249                          rsc->id, pe__node_name(node));
1250             pe__set_action_flags(promote, pe_action_pseudo);
1251         }
1252     } else {
1253         pe_rsc_trace(rsc, "Not promoting %s on %s: start unrunnable",
1254                      rsc->id, pe__node_name(node));
1255         action_list = pe__resource_actions(rsc, node, RSC_PROMOTE, true);
1256         for (iter = action_list; iter != NULL; iter = iter->next) {
1257             pe_action_t *promote = (pe_action_t *) iter->data;
1258 
1259             pe__clear_action_flags(promote, pe_action_runnable);
1260         }
1261         g_list_free(action_list);
1262     }
1263 }
1264 
1265 /*!
1266  * \internal
1267  * \brief Schedule actions needed to demote a resource wherever it is active
1268  *
1269  * \param[in,out] rsc       Resource being demoted
1270  * \param[in]     node      Node where resource should be demoted (ignored)
1271  * \param[in]     optional  Whether actions should be optional
1272  */
1273 static void
1274 demote_resource(pe_resource_t *rsc, pe_node_t *node, bool optional)
     /*  */
1275 {
1276     /* Since this will only be called for a primitive (possibly as an instance
1277      * of a collective resource), the resource is multiply active if it is
1278      * running on more than one node, so we want to demote on all of them as
1279      * part of recovery, regardless of which one is the desired node.
1280      */
1281     for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
1282         pe_node_t *current = (pe_node_t *) iter->data;
1283 
1284         if (is_expected_node(rsc, current)) {
1285             pe_rsc_trace(rsc,
1286                          "Skipping demote of multiply active resource %s "
1287                          "on expected node %s",
1288                          rsc->id, pe__node_name(current));
1289         } else {
1290             pe_rsc_trace(rsc, "Scheduling %s demotion of %s on %s",
1291                          (optional? "optional" : "required"), rsc->id,
1292                          pe__node_name(current));
1293             demote_action(rsc, current, optional);
1294         }
1295     }
1296 }
1297 
1298 static void
1299 assert_role_error(pe_resource_t *rsc, pe_node_t *node, bool optional)
     /*  */
1300 {
1301     CRM_ASSERT(false);
1302 }
1303 
1304 /*!
1305  * \internal
1306  * \brief Schedule cleanup of a resource
1307  *
1308  * \param[in,out] rsc       Resource to clean up
1309  * \param[in]     node      Node to clean up on
1310  * \param[in]     optional  Whether clean-up should be optional
1311  */
1312 void
1313 pcmk__schedule_cleanup(pe_resource_t *rsc, const pe_node_t *node, bool optional)
     /*  */
1314 {
1315     /* If the cleanup is required, its orderings are optional, because they're
1316      * relevant only if both actions are required. Conversely, if the cleanup is
1317      * optional, the orderings make the then action required if the first action
1318      * becomes required.
1319      */
1320     uint32_t flag = optional? pe_order_implies_then : pe_order_optional;
1321 
1322     CRM_CHECK((rsc != NULL) && (node != NULL), return);
1323 
1324     if (pcmk_is_set(rsc->flags, pe_rsc_failed)) {
1325         pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: resource failed",
1326                      rsc->id, pe__node_name(node));
1327         return;
1328     }
1329 
1330     if (node->details->unclean || !node->details->online) {
1331         pe_rsc_trace(rsc, "Skipping clean-up of %s on %s: node unavailable",
1332                      rsc->id, pe__node_name(node));
1333         return;
1334     }
1335 
1336     crm_notice("Scheduling clean-up of %s on %s", rsc->id, pe__node_name(node));
1337     delete_action(rsc, node, optional);
1338 
1339     // stop -> clean-up -> start
1340     pcmk__order_resource_actions(rsc, RSC_STOP, rsc, RSC_DELETE, flag);
1341     pcmk__order_resource_actions(rsc, RSC_DELETE, rsc, RSC_START, flag);
1342 }
1343 
1344 /*!
1345  * \internal
1346  * \brief Add primitive meta-attributes relevant to graph actions to XML
1347  *
1348  * \param[in]     rsc  Primitive resource whose meta-attributes should be added
1349  * \param[in,out] xml  Transition graph action attributes XML to add to
1350  */
1351 void
1352 pcmk__primitive_add_graph_meta(pe_resource_t *rsc, xmlNode *xml)
     /*  */
1353 {
1354     char *name = NULL;
1355     char *value = NULL;
1356     const pe_resource_t *parent = NULL;
1357 
1358     CRM_ASSERT((rsc != NULL) && (xml != NULL));
1359 
1360     /* Clone instance numbers get set internally as meta-attributes, and are
1361      * needed in the transition graph (for example, to tell unique clone
1362      * instances apart).
1363      */
1364     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION);
1365     if (value != NULL) {
1366         name = crm_meta_name(XML_RSC_ATTR_INCARNATION);
1367         crm_xml_add(xml, name, value);
1368         free(name);
1369     }
1370 
1371     // Not sure if this one is really needed ...
1372     value = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_REMOTE_NODE);
1373     if (value != NULL) {
1374         name = crm_meta_name(XML_RSC_ATTR_REMOTE_NODE);
1375         crm_xml_add(xml, name, value);
1376         free(name);
1377     }
1378 
1379     /* The container meta-attribute can be set on the primitive itself or one of
1380      * its parents (for example, a group inside a container resource), so check
1381      * them all, and keep the highest one found.
1382      */
1383     for (parent = rsc; parent != NULL; parent = parent->parent) {
1384         if (parent->container != NULL) {
1385             crm_xml_add(xml, CRM_META "_" XML_RSC_ATTR_CONTAINER,
1386                         parent->container->id);
1387         }
1388     }
1389 
1390     /* Bundle replica children will get their external-ip set internally as a
1391      * meta-attribute. The graph action needs it, but under a different naming
1392      * convention than other meta-attributes.
1393      */
1394     value = g_hash_table_lookup(rsc->meta, "external-ip");
1395     if (value != NULL) {
1396         crm_xml_add(xml, "pcmk_external_ip", value);
1397     }
1398 }
1399 
1400 // Primitive implementation of resource_alloc_functions_t:add_utilization()
1401 void
1402 pcmk__primitive_add_utilization(const pe_resource_t *rsc,
     /*  */
1403                                 const pe_resource_t *orig_rsc, GList *all_rscs,
1404                                 GHashTable *utilization)
1405 {
1406     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
1407         return;
1408     }
1409 
1410     pe_rsc_trace(orig_rsc, "%s: Adding primitive %s as colocated utilization",
1411                  orig_rsc->id, rsc->id);
1412     pcmk__release_node_capacity(utilization, rsc);
1413 }
1414 
1415 /*!
1416  * \internal
1417  * \brief Get epoch time of node's shutdown attribute (or now if none)
1418  *
1419  * \param[in] node      Node to check
1420  * \param[in] data_set  Cluster working set
1421  *
1422  * \return Epoch time corresponding to shutdown attribute if set or now if not
1423  */
1424 static time_t
1425 shutdown_time(const pe_node_t *node)
     /*  */
1426 {
1427     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
1428     time_t result = 0;
1429 
1430     if (shutdown != NULL) {
1431         long long result_ll;
1432 
1433         if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
1434             result = (time_t) result_ll;
1435         }
1436     }
1437     return (result == 0)? get_effective_time(node->details->data_set) : result;
1438 }
1439 
1440 /*!
1441  * \internal
1442  * \brief Ban a resource from a node if it's not locked to the node
1443  *
1444  * \param[in] data  Node to check
1445  * \param[in] user_data  Resource to check
1446  */
1447 static void
1448 ban_if_not_locked(gpointer data, gpointer user_data)
     /*  */
1449 {
1450     pe_node_t *node = (pe_node_t *) data;
1451     pe_resource_t *rsc = (pe_resource_t *) user_data;
1452 
1453     if (strcmp(node->details->uname, rsc->lock_node->details->uname) != 0) {
1454         resource_location(rsc, node, -CRM_SCORE_INFINITY,
1455                           XML_CONFIG_ATTR_SHUTDOWN_LOCK, rsc->cluster);
1456     }
1457 }
1458 
1459 // Primitive implementation of resource_alloc_functions_t:shutdown_lock()
1460 void
1461 pcmk__primitive_shutdown_lock(pe_resource_t *rsc)
     /*  */
1462 {
1463     const char *class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
1464 
1465     // Fence devices and remote connections can't be locked
1466     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
1467         || pe__resource_is_remote_conn(rsc, rsc->cluster)) {
1468         return;
1469     }
1470 
1471     if (rsc->lock_node != NULL) {
1472         // The lock was obtained from resource history
1473 
1474         if (rsc->running_on != NULL) {
1475             /* The resource was started elsewhere even though it is now
1476              * considered locked. This shouldn't be possible, but as a
1477              * failsafe, we don't want to disturb the resource now.
1478              */
1479             pe_rsc_info(rsc,
1480                         "Cancelling shutdown lock because %s is already active",
1481                         rsc->id);
1482             pe__clear_resource_history(rsc, rsc->lock_node, rsc->cluster);
1483             rsc->lock_node = NULL;
1484             rsc->lock_time = 0;
1485         }
1486 
1487     // Only a resource active on exactly one node can be locked
1488     } else if (pcmk__list_of_1(rsc->running_on)) {
1489         pe_node_t *node = rsc->running_on->data;
1490 
1491         if (node->details->shutdown) {
1492             if (node->details->unclean) {
1493                 pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
1494                              rsc->id, pe__node_name(node));
1495             } else {
1496                 rsc->lock_node = node;
1497                 rsc->lock_time = shutdown_time(node);
1498             }
1499         }
1500     }
1501 
1502     if (rsc->lock_node == NULL) {
1503         // No lock needed
1504         return;
1505     }
1506 
1507     if (rsc->cluster->shutdown_lock > 0) {
1508         time_t lock_expiration = rsc->lock_time + rsc->cluster->shutdown_lock;
1509 
1510         pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
1511                     rsc->id, pe__node_name(rsc->lock_node),
1512                     (long long) lock_expiration);
1513         pe__update_recheck_time(++lock_expiration, rsc->cluster);
1514     } else {
1515         pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
1516                     rsc->id, pe__node_name(rsc->lock_node));
1517     }
1518 
1519     // If resource is locked to one node, ban it from all other nodes
1520     g_list_foreach(rsc->cluster->nodes, ban_if_not_locked, rsc);
1521 }
/* */
root/lib/pacemaker/pcmk_sched_primitive.c

DEFINITIONS
