root/lib/pacemaker/pcmk_sched_group.c

/* */

DEFINITIONS

1. pcmk__group_assign
2. create_group_pseudo_op
3. pcmk__group_create_actions
4. member_internal_constraints
5. pcmk__group_internal_constraints
6. colocate_group_with
7. colocate_with_group
8. pcmk__group_apply_coloc_score
9. pcmk__group_action_flags
10. pcmk__group_update_ordered_actions
11. pcmk__group_apply_location
12. pcmk__group_colocated_resources
13. pcmk__with_group_colocations
14. pcmk__group_with_colocations
15. pcmk__group_add_colocated_node_scores
16. pcmk__group_add_utilization
17. pcmk__group_shutdown_lock

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdbool.h>
  13 
  14 #include <qb/qbdefs.h>              // QB_ABS()
  15 
  16 #include <crm/common/xml.h>
  17 
  18 #include <pacemaker-internal.h>
  19 #include "libpacemaker_private.h"
  20 
  21 /*!
  22  * \internal
  23  * \brief Assign a group resource to a node
  24  *
  25  * \param[in,out] rsc           Group resource to assign to a node
  26  * \param[in]     prefer        Node to prefer, if all else is equal
  27  * \param[in]     stop_if_fail  If \c true and a child of \p rsc can't be
  28  *                              assigned to a node, set the child's next role to
  29  *                              stopped and update existing actions
  30  *
  31  * \return Node that \p rsc is assigned to, if assigned entirely to one node
  32  *
  33  * \note If \p stop_if_fail is \c false, then \c pcmk__unassign_resource() can
  34  *       completely undo the assignment. A successful assignment can be either
  35  *       undone or left alone as final. A failed assignment has the same effect
  36  *       as calling pcmk__unassign_resource(); there are no side effects on
  37  *       roles or actions.
  38  */
  39 pcmk_node_t *
  40 pcmk__group_assign(pcmk_resource_t *rsc, const pcmk_node_t *prefer,
     /*  */
  41                    bool stop_if_fail)
  42 {
  43     pcmk_node_t *first_assigned_node = NULL;
  44     pcmk_resource_t *first_member = NULL;
  45 
  46     CRM_ASSERT(pcmk__is_group(rsc));
  47 
  48     if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
  49         return rsc->allocated_to; // Assignment already done
  50     }
  51     if (pcmk_is_set(rsc->flags, pcmk_rsc_assigning)) {
  52         pcmk__rsc_debug(rsc, "Assignment dependency loop detected involving %s",
  53                         rsc->id);
  54         return NULL;
  55     }
  56 
  57     if (rsc->children == NULL) {
  58         // No members to assign
  59         pcmk__clear_rsc_flags(rsc, pcmk_rsc_unassigned);
  60         return NULL;
  61     }
  62 
  63     pcmk__set_rsc_flags(rsc, pcmk_rsc_assigning);
  64     first_member = (pcmk_resource_t *) rsc->children->data;
  65     rsc->role = first_member->role;
  66 
  67     pe__show_node_scores(!pcmk_is_set(rsc->cluster->flags,
  68                                       pcmk_sched_output_scores),
  69                          rsc, __func__, rsc->allowed_nodes, rsc->cluster);
  70 
  71     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
  72         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
  73         pcmk_node_t *node = NULL;
  74 
  75         pcmk__rsc_trace(rsc, "Assigning group %s member %s",
  76                         rsc->id, member->id);
  77         node = member->cmds->assign(member, prefer, stop_if_fail);
  78         if (first_assigned_node == NULL) {
  79             first_assigned_node = node;
  80         }
  81     }
  82 
  83     pe__set_next_role(rsc, first_member->next_role, "first group member");
  84     pcmk__clear_rsc_flags(rsc, pcmk_rsc_assigning|pcmk_rsc_unassigned);
  85 
  86     if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
  87         return NULL;
  88     }
  89     return first_assigned_node;
  90 }
  91 
  92 /*!
  93  * \internal
  94  * \brief Create a pseudo-operation for a group as an ordering point
  95  *
  96  * \param[in,out] group   Group resource to create action for
  97  * \param[in]     action  Action name
  98  *
  99  * \return Newly created pseudo-operation
 100  */
 101 static pcmk_action_t *
 102 create_group_pseudo_op(pcmk_resource_t *group, const char *action)
     /*  */
 103 {
 104     pcmk_action_t *op = custom_action(group, pcmk__op_key(group->id, action, 0),
 105                                       action, NULL, TRUE, group->cluster);
 106 
 107     pcmk__set_action_flags(op, pcmk_action_pseudo|pcmk_action_runnable);
 108     return op;
 109 }
 110 
 111 /*!
 112  * \internal
 113  * \brief Create all actions needed for a given group resource
 114  *
 115  * \param[in,out] rsc  Group resource to create actions for
 116  */
 117 void
 118 pcmk__group_create_actions(pcmk_resource_t *rsc)
     /*  */
 119 {
 120     CRM_ASSERT(pcmk__is_group(rsc));
 121 
 122     pcmk__rsc_trace(rsc, "Creating actions for group %s", rsc->id);
 123 
 124     // Create actions for individual group members
 125     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 126         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
 127 
 128         member->cmds->create_actions(member);
 129     }
 130 
 131     // Create pseudo-actions for group itself to serve as ordering points
 132     create_group_pseudo_op(rsc, PCMK_ACTION_START);
 133     create_group_pseudo_op(rsc, PCMK_ACTION_RUNNING);
 134     create_group_pseudo_op(rsc, PCMK_ACTION_STOP);
 135     create_group_pseudo_op(rsc, PCMK_ACTION_STOPPED);
 136     if (crm_is_true(g_hash_table_lookup(rsc->meta, PCMK_META_PROMOTABLE))) {
 137         create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTE);
 138         create_group_pseudo_op(rsc, PCMK_ACTION_DEMOTED);
 139         create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTE);
 140         create_group_pseudo_op(rsc, PCMK_ACTION_PROMOTED);
 141     }
 142 }
 143 
 144 // User data for member_internal_constraints()
 145 struct member_data {
 146     // These could be derived from member but this avoids some function calls
 147     bool ordered;
 148     bool colocated;
 149     bool promotable;
 150 
 151     pcmk_resource_t *last_active;
 152     pcmk_resource_t *previous_member;
 153 };
 154 
 155 /*!
 156  * \internal
 157  * \brief Create implicit constraints needed for a group member
 158  *
 159  * \param[in,out] data       Group member to create implicit constraints for
 160  * \param[in,out] user_data  Member data (struct member_data *)
 161  */
 162 static void
 163 member_internal_constraints(gpointer data, gpointer user_data)
     /*  */
 164 {
 165     pcmk_resource_t *member = (pcmk_resource_t *) data;
 166     struct member_data *member_data = (struct member_data *) user_data;
 167 
 168     // For ordering demote vs demote or stop vs stop
 169     uint32_t down_flags = pcmk__ar_then_implies_first_graphed;
 170 
 171     // For ordering demote vs demoted or stop vs stopped
 172     uint32_t post_down_flags = pcmk__ar_first_implies_then_graphed;
 173 
 174     // Create the individual member's implicit constraints
 175     member->cmds->internal_constraints(member);
 176 
 177     if (member_data->previous_member == NULL) {
 178         // This is first member
 179         if (member_data->ordered) {
 180             pcmk__set_relation_flags(down_flags, pcmk__ar_ordered);
 181             post_down_flags = pcmk__ar_first_implies_then;
 182         }
 183 
 184     } else if (member_data->colocated) {
 185         uint32_t flags = pcmk__coloc_none;
 186 
 187         if (pcmk_is_set(member->flags, pcmk_rsc_critical)) {
 188             flags |= pcmk__coloc_influence;
 189         }
 190 
 191         // Colocate this member with the previous one
 192         pcmk__new_colocation("#group-members", NULL, PCMK_SCORE_INFINITY,
 193                              member, member_data->previous_member, NULL, NULL,
 194                              flags);
 195     }
 196 
 197     if (member_data->promotable) {
 198         // Demote group -> demote member -> group is demoted
 199         pcmk__order_resource_actions(member->parent, PCMK_ACTION_DEMOTE,
 200                                      member, PCMK_ACTION_DEMOTE, down_flags);
 201         pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
 202                                      member->parent, PCMK_ACTION_DEMOTED,
 203                                      post_down_flags);
 204 
 205         // Promote group -> promote member -> group is promoted
 206         pcmk__order_resource_actions(member, PCMK_ACTION_PROMOTE,
 207                                      member->parent, PCMK_ACTION_PROMOTED,
 208                                      pcmk__ar_unrunnable_first_blocks
 209                                      |pcmk__ar_first_implies_then
 210                                      |pcmk__ar_first_implies_then_graphed);
 211         pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
 212                                      member, PCMK_ACTION_PROMOTE,
 213                                      pcmk__ar_then_implies_first_graphed);
 214     }
 215 
 216     // Stop group -> stop member -> group is stopped
 217     pcmk__order_stops(member->parent, member, down_flags);
 218     pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
 219                                  member->parent, PCMK_ACTION_STOPPED,
 220                                  post_down_flags);
 221 
 222     // Start group -> start member -> group is started
 223     pcmk__order_starts(member->parent, member,
 224                        pcmk__ar_then_implies_first_graphed);
 225     pcmk__order_resource_actions(member, PCMK_ACTION_START,
 226                                  member->parent, PCMK_ACTION_RUNNING,
 227                                  pcmk__ar_unrunnable_first_blocks
 228                                  |pcmk__ar_first_implies_then
 229                                  |pcmk__ar_first_implies_then_graphed);
 230 
 231     if (!member_data->ordered) {
 232         pcmk__order_starts(member->parent, member,
 233                            pcmk__ar_first_implies_then
 234                            |pcmk__ar_unrunnable_first_blocks
 235                            |pcmk__ar_then_implies_first_graphed);
 236         if (member_data->promotable) {
 237             pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
 238                                          member, PCMK_ACTION_PROMOTE,
 239                                          pcmk__ar_first_implies_then
 240                                          |pcmk__ar_unrunnable_first_blocks
 241                                          |pcmk__ar_then_implies_first_graphed);
 242         }
 243 
 244     } else if (member_data->previous_member == NULL) {
 245         pcmk__order_starts(member->parent, member, pcmk__ar_none);
 246         if (member_data->promotable) {
 247             pcmk__order_resource_actions(member->parent, PCMK_ACTION_PROMOTE,
 248                                          member, PCMK_ACTION_PROMOTE,
 249                                          pcmk__ar_none);
 250         }
 251 
 252     } else {
 253         // Order this member relative to the previous one
 254 
 255         pcmk__order_starts(member_data->previous_member, member,
 256                            pcmk__ar_first_implies_then
 257                            |pcmk__ar_unrunnable_first_blocks);
 258         pcmk__order_stops(member, member_data->previous_member,
 259                           pcmk__ar_ordered|pcmk__ar_intermediate_stop);
 260 
 261         /* In unusual circumstances (such as adding a new member to the middle
 262          * of a group with unmanaged later members), this member may be active
 263          * while the previous (new) member is inactive. In this situation, the
 264          * usual restart orderings will be irrelevant, so we need to order this
 265          * member's stop before the previous member's start.
 266          */
 267         if ((member->running_on != NULL)
 268             && (member_data->previous_member->running_on == NULL)) {
 269             pcmk__order_resource_actions(member, PCMK_ACTION_STOP,
 270                                          member_data->previous_member,
 271                                          PCMK_ACTION_START,
 272                                          pcmk__ar_then_implies_first
 273                                          |pcmk__ar_unrunnable_first_blocks);
 274         }
 275 
 276         if (member_data->promotable) {
 277             pcmk__order_resource_actions(member_data->previous_member,
 278                                          PCMK_ACTION_PROMOTE, member,
 279                                          PCMK_ACTION_PROMOTE,
 280                                          pcmk__ar_first_implies_then
 281                                          |pcmk__ar_unrunnable_first_blocks);
 282             pcmk__order_resource_actions(member, PCMK_ACTION_DEMOTE,
 283                                          member_data->previous_member,
 284                                          PCMK_ACTION_DEMOTE, pcmk__ar_ordered);
 285         }
 286     }
 287 
 288     // Make sure partially active groups shut down in sequence
 289     if (member->running_on != NULL) {
 290         if (member_data->ordered && (member_data->previous_member != NULL)
 291             && (member_data->previous_member->running_on == NULL)
 292             && (member_data->last_active != NULL)
 293             && (member_data->last_active->running_on != NULL)) {
 294             pcmk__order_stops(member, member_data->last_active,
 295                               pcmk__ar_ordered);
 296         }
 297         member_data->last_active = member;
 298     }
 299 
 300     member_data->previous_member = member;
 301 }
 302 
 303 /*!
 304  * \internal
 305  * \brief Create implicit constraints needed for a group resource
 306  *
 307  * \param[in,out] rsc  Group resource to create implicit constraints for
 308  */
 309 void
 310 pcmk__group_internal_constraints(pcmk_resource_t *rsc)
     /*  */
 311 {
 312     struct member_data member_data = { false, };
 313     const pcmk_resource_t *top = NULL;
 314 
 315     CRM_ASSERT(pcmk__is_group(rsc));
 316 
 317     /* Order group pseudo-actions relative to each other for restarting:
 318      * stop group -> group is stopped -> start group -> group is started
 319      */
 320     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOP,
 321                                  rsc, PCMK_ACTION_STOPPED,
 322                                  pcmk__ar_unrunnable_first_blocks);
 323     pcmk__order_resource_actions(rsc, PCMK_ACTION_STOPPED,
 324                                  rsc, PCMK_ACTION_START,
 325                                  pcmk__ar_ordered);
 326     pcmk__order_resource_actions(rsc, PCMK_ACTION_START,
 327                                  rsc, PCMK_ACTION_RUNNING,
 328                                  pcmk__ar_unrunnable_first_blocks);
 329 
 330     top = pe__const_top_resource(rsc, false);
 331 
 332     member_data.ordered = pe__group_flag_is_set(rsc, pcmk__group_ordered);
 333     member_data.colocated = pe__group_flag_is_set(rsc, pcmk__group_colocated);
 334     member_data.promotable = pcmk_is_set(top->flags, pcmk_rsc_promotable);
 335     g_list_foreach(rsc->children, member_internal_constraints, &member_data);
 336 }
 337 
 338 /*!
 339  * \internal
 340  * \brief Apply a colocation's score to node scores or resource priority
 341  *
 342  * Given a colocation constraint for a group with some other resource, apply the
 343  * score to the dependent's allowed node scores (if we are still placing
 344  * resources) or priority (if we are choosing promotable clone instance roles).
 345  *
 346  * \param[in,out] dependent      Dependent group resource in colocation
 347  * \param[in]     primary        Primary resource in colocation
 348  * \param[in]     colocation     Colocation constraint to apply
 349  *
 350  * \return The score added to the dependent's priority
 351  */
 352 static int
 353 colocate_group_with(pcmk_resource_t *dependent, const pcmk_resource_t *primary,
     /*  */
 354                     const pcmk__colocation_t *colocation)
 355 {
 356     int priority_delta = 0;
 357 
 358     if (dependent->children == NULL) {
 359         return 0;
 360     }
 361 
 362     pcmk__rsc_trace(primary, "Processing %s (group %s with %s) for dependent",
 363                     colocation->id, dependent->id, primary->id);
 364 
 365     if (pe__group_flag_is_set(dependent, pcmk__group_colocated)) {
 366         // Colocate first member (internal colocations will handle the rest)
 367         pcmk_resource_t *member = dependent->children->data;
 368 
 369         priority_delta = member->cmds->apply_coloc_score(member, primary,
 370                                                          colocation, true);
 371 
 372     } else {
 373         if (colocation->score >= PCMK_SCORE_INFINITY) {
 374             pcmk__config_err("%s: Cannot perform mandatory colocation between "
 375                              "non-colocated group and %s",
 376                              dependent->id, primary->id);
 377             return 0;
 378         }
 379 
 380         // Colocate each member individually
 381         for (GList *iter = dependent->children; iter != NULL;
 382              iter = iter->next) {
 383 
 384             pcmk_resource_t *member = iter->data;
 385             int instance_delta = member->cmds->apply_coloc_score(member,
 386                                                                  primary,
 387                                                                  colocation,
 388                                                                  false);
 389 
 390             /* priority_delta is used for determining which instances of a
 391              * promotable clone to promote. It's possible that colocations
 392              * involving promotable cloned non-colocated groups may not behave
 393              * correctly in all circumstances. Non-colocated groups are
 394              * deprecated, and testing focused on colocated groups.
 395              */
 396             priority_delta = pcmk__add_scores(priority_delta, instance_delta);
 397         }
 398     }
 399 
 400     if (priority_delta != 0) {
 401         dependent->priority = pcmk__add_scores(priority_delta,
 402                                                dependent->priority);
 403 
 404         pcmk__rsc_trace(dependent,
 405                         "Applied %s to %s promotion priority "
 406                         "(now %s after %s %d)",
 407                         colocation->id, dependent->id,
 408                         pcmk_readable_score(dependent->priority),
 409                         ((priority_delta > 0)? "adding" : "subtracting"),
 410                         QB_ABS(priority_delta));
 411     }
 412     return priority_delta;
 413 }
 414 
 415 /*!
 416  * \internal
 417  * \brief Apply a colocation's score to node scores or resource priority
 418  *
 419  * Given a colocation constraint for some other resource with a group, apply the
 420  * score to the dependent's allowed node scores (if we are still placing
 421  * resources) or priority (if we are choosing promotable clone instance roles).
 422  *
 423  * \param[in,out] dependent      Dependent resource in colocation
 424  * \param[in]     primary        Primary group resource in colocation
 425  * \param[in]     colocation     Colocation constraint to apply
 426  *
 427  * \return The score added to the dependent's priority
 428  */
 429 static int
 430 colocate_with_group(pcmk_resource_t *dependent, const pcmk_resource_t *primary,
     /*  */
 431                     const pcmk__colocation_t *colocation)
 432 {
 433     int priority_delta = 0;
 434     const pcmk_resource_t *member = NULL;
 435 
 436     pcmk__rsc_trace(primary,
 437                     "Processing colocation %s (%s with group %s) for primary",
 438                     colocation->id, dependent->id, primary->id);
 439 
 440     if (pcmk_is_set(primary->flags, pcmk_rsc_unassigned)) {
 441         return 0;
 442     }
 443 
 444     if (pe__group_flag_is_set(primary, pcmk__group_colocated)) {
 445 
 446         if (colocation->score >= PCMK_SCORE_INFINITY) {
 447             /* For mandatory colocations, the entire group must be assignable
 448              * (and in the specified role if any), so apply the colocation based
 449              * on the last member.
 450              */
 451             member = pe__last_group_member(primary);
 452         } else if (primary->children != NULL) {
 453             /* For optional colocations, whether the group is partially or fully
 454              * up doesn't matter, so apply the colocation based on the first
 455              * member.
 456              */
 457             member = (pcmk_resource_t *) primary->children->data;
 458         }
 459         if (member == NULL) {
 460             return 0;   // Nothing to colocate with
 461         }
 462 
 463         return member->cmds->apply_coloc_score(dependent, member, colocation,
 464                                                false);
 465     }
 466 
 467     if (colocation->score >= PCMK_SCORE_INFINITY) {
 468         pcmk__config_err("%s: Cannot perform mandatory colocation with"
 469                          " non-colocated group %s",
 470                          dependent->id, primary->id);
 471         return 0;
 472     }
 473 
 474     // Colocate dependent with each member individually
 475     for (const GList *iter = primary->children; iter != NULL;
 476          iter = iter->next) {
 477 
 478         int instance_delta = 0;
 479 
 480         member = iter->data;
 481         instance_delta = member->cmds->apply_coloc_score(dependent, member,
 482                                                          colocation, false);
 483         priority_delta = pcmk__add_scores(priority_delta, instance_delta);
 484     }
 485     return priority_delta;
 486 }
 487 
 488 /*!
 489  * \internal
 490  * \brief Apply a colocation's score to node scores or resource priority
 491  *
 492  * Given a colocation constraint, apply its score to the dependent's
 493  * allowed node scores (if we are still placing resources) or priority (if
 494  * we are choosing promotable clone instance roles).
 495  *
 496  * \param[in,out] dependent      Dependent resource in colocation
 497  * \param[in]     primary        Primary resource in colocation
 498  * \param[in]     colocation     Colocation constraint to apply
 499  * \param[in]     for_dependent  true if called on behalf of dependent
 500  *
 501  * \return The score added to the dependent's priority
 502  */
 503 int
 504 pcmk__group_apply_coloc_score(pcmk_resource_t *dependent,
     /*  */
 505                               const pcmk_resource_t *primary,
 506                               const pcmk__colocation_t *colocation,
 507                               bool for_dependent)
 508 {
 509     CRM_ASSERT((dependent != NULL) && (primary != NULL)
 510                && (colocation != NULL));
 511 
 512     if (for_dependent) {
 513         return colocate_group_with(dependent, primary, colocation);
 514 
 515     } else {
 516         // Method should only be called for primitive dependents
 517         CRM_ASSERT(pcmk__is_primitive(dependent));
 518 
 519         return colocate_with_group(dependent, primary, colocation);
 520     }
 521 }
 522 
 523 /*!
 524  * \internal
 525  * \brief Return action flags for a given group resource action
 526  *
 527  * \param[in,out] action  Group action to get flags for
 528  * \param[in]     node    If not NULL, limit effects to this node
 529  *
 530  * \return Flags appropriate to \p action on \p node
 531  */
 532 uint32_t
 533 pcmk__group_action_flags(pcmk_action_t *action, const pcmk_node_t *node)
     /*  */
 534 {
 535     // Default flags for a group action
 536     uint32_t flags = pcmk_action_optional
 537                      |pcmk_action_runnable
 538                      |pcmk_action_pseudo;
 539 
 540     CRM_ASSERT(action != NULL);
 541 
 542     // Update flags considering each member's own flags for same action
 543     for (GList *iter = action->rsc->children; iter != NULL; iter = iter->next) {
 544         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
 545 
 546         // Check whether member has the same action
 547         enum action_tasks task = get_complex_task(member, action->task);
 548         const char *task_s = pcmk_action_text(task);
 549         pcmk_action_t *member_action = find_first_action(member->actions, NULL,
 550                                                          task_s, node);
 551 
 552         if (member_action != NULL) {
 553             uint32_t member_flags = member->cmds->action_flags(member_action,
 554                                                                node);
 555 
 556             // Group action is mandatory if any member action is
 557             if (pcmk_is_set(flags, pcmk_action_optional)
 558                 && !pcmk_is_set(member_flags, pcmk_action_optional)) {
 559                 pcmk__rsc_trace(action->rsc, "%s is mandatory because %s is",
 560                                 action->uuid, member_action->uuid);
 561                 pcmk__clear_raw_action_flags(flags, "group action",
 562                                              pcmk_action_optional);
 563                 pcmk__clear_action_flags(action, pcmk_action_optional);
 564             }
 565 
 566             // Group action is unrunnable if any member action is
 567             if (!pcmk__str_eq(task_s, action->task, pcmk__str_none)
 568                 && pcmk_is_set(flags, pcmk_action_runnable)
 569                 && !pcmk_is_set(member_flags, pcmk_action_runnable)) {
 570 
 571                 pcmk__rsc_trace(action->rsc, "%s is unrunnable because %s is",
 572                                 action->uuid, member_action->uuid);
 573                 pcmk__clear_raw_action_flags(flags, "group action",
 574                                              pcmk_action_runnable);
 575                 pcmk__clear_action_flags(action, pcmk_action_runnable);
 576             }
 577 
 578         /* Group (pseudo-)actions other than stop or demote are unrunnable
 579          * unless every member will do it.
 580          */
 581         } else if ((task != pcmk_action_stop) && (task != pcmk_action_demote)) {
 582             pcmk__rsc_trace(action->rsc,
 583                             "%s is not runnable because %s will not %s",
 584                             action->uuid, member->id, task_s);
 585             pcmk__clear_raw_action_flags(flags, "group action",
 586                                          pcmk_action_runnable);
 587         }
 588     }
 589 
 590     return flags;
 591 }
 592 
 593 /*!
 594  * \internal
 595  * \brief Update two actions according to an ordering between them
 596  *
 597  * Given information about an ordering of two actions, update the actions' flags
 598  * (and runnable_before members if appropriate) as appropriate for the ordering.
 599  * Effects may cascade to other orderings involving the actions as well.
 600  *
 601  * \param[in,out] first      'First' action in an ordering
 602  * \param[in,out] then       'Then' action in an ordering
 603  * \param[in]     node       If not NULL, limit scope of ordering to this node
 604  *                           (only used when interleaving instances)
 605  * \param[in]     flags      Action flags for \p first for ordering purposes
 606  * \param[in]     filter     Action flags to limit scope of certain updates (may
 607  *                           include pcmk_action_optional to affect only
 608  *                           mandatory actions, and pcmk_action_runnable to
 609  *                           affect only runnable actions)
 610  * \param[in]     type       Group of enum pcmk__action_relation_flags to apply
 611  * \param[in,out] scheduler  Scheduler data
 612  *
 613  * \return Group of enum pcmk__updated flags indicating what was updated
 614  */
 615 uint32_t
 616 pcmk__group_update_ordered_actions(pcmk_action_t *first, pcmk_action_t *then,
     /*  */
 617                                    const pcmk_node_t *node, uint32_t flags,
 618                                    uint32_t filter, uint32_t type,
 619                                    pcmk_scheduler_t *scheduler)
 620 {
 621     uint32_t changed = pcmk__updated_none;
 622 
 623     // Group method can be called only on behalf of "then" action
 624     CRM_ASSERT((first != NULL) && (then != NULL) && (then->rsc != NULL)
 625                && (scheduler != NULL));
 626 
 627     // Update the actions for the group itself
 628     changed |= pcmk__update_ordered_actions(first, then, node, flags, filter,
 629                                             type, scheduler);
 630 
 631     // Update the actions for each group member
 632     for (GList *iter = then->rsc->children; iter != NULL; iter = iter->next) {
 633         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
 634 
 635         pcmk_action_t *member_action = find_first_action(member->actions, NULL,
 636                                                          then->task, node);
 637 
 638         if (member_action != NULL) {
 639             changed |= member->cmds->update_ordered_actions(first,
 640                                                             member_action, node,
 641                                                             flags, filter, type,
 642                                                             scheduler);
 643         }
 644     }
 645     return changed;
 646 }
 647 
 648 /*!
 649  * \internal
 650  * \brief Apply a location constraint to a group's allowed node scores
 651  *
 652  * \param[in,out] rsc       Group resource to apply constraint to
 653  * \param[in,out] location  Location constraint to apply
 654  */
 655 void
 656 pcmk__group_apply_location(pcmk_resource_t *rsc, pcmk__location_t *location)
     /*  */
 657 {
 658     GList *node_list_orig = NULL;
 659     GList *node_list_copy = NULL;
 660 
 661     CRM_ASSERT(pcmk__is_group(rsc) && (location != NULL));
 662 
 663     // Save the constraint's original node list (with the constraint score)
 664     node_list_orig = location->nodes;
 665 
 666     // Make a copy of the nodes with all zero scores
 667     node_list_copy  = pcmk__copy_node_list(node_list_orig, true);
 668 
 669     /* Apply the constraint to the group itself. This ensures that any nodes
 670      * affected by the constraint are in the group's allowed nodes, with the
 671      * constraint score added.
 672      */
 673     pcmk__apply_location(rsc, location);
 674 
 675     // Apply the constraint for each member
 676     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 677         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
 678 
 679         if (pe__group_flag_is_set(rsc, pcmk__group_colocated)
 680             && (iter != rsc->children)) {
 681             /* When apply_location() is called below for the first member (iter
 682              * == rsc->children), the constraint score will be added to
 683              * the member's affected allowed nodes.
 684              *
 685              * For subsequent members, we reset the constraint's node table to
 686              * the copy with all 0 scores. Otherwise, when assigning the member,
 687              * the constraint score would be counted multiple times (once for
 688              * each later member) due to internal group colocations. Though the
 689              * 0 score will not affect these members' allowed node scores, it
 690              * ensures that affected nodes are in each member's allowed nodes,
 691              * enabling the member on those nodes in asymmetric clusters.
 692              */
 693             location->nodes = node_list_copy;
 694         }
 695 
 696         member->cmds->apply_location(member, location);
 697     }
 698 
 699     location->nodes = node_list_orig;
 700     g_list_free_full(node_list_copy, free);
 701 }
 702 
 703 // Group implementation of pcmk_assignment_methods_t:colocated_resources()
 704 GList *
 705 pcmk__group_colocated_resources(const pcmk_resource_t *rsc,
     /*  */
 706                                 const pcmk_resource_t *orig_rsc,
 707                                 GList *colocated_rscs)
 708 {
 709     const pcmk_resource_t *member = NULL;
 710 
 711     CRM_ASSERT(pcmk__is_group(rsc));
 712 
 713     if (orig_rsc == NULL) {
 714         orig_rsc = rsc;
 715     }
 716 
 717     if (pe__group_flag_is_set(rsc, pcmk__group_colocated)
 718         || pcmk__is_clone(rsc->parent)) {
 719         /* This group has colocated members and/or is cloned -- either way,
 720          * add every child's colocated resources to the list. The first and last
 721          * members will include the group's own colocations.
 722          */
 723         colocated_rscs = g_list_prepend(colocated_rscs, (gpointer) rsc);
 724         for (const GList *iter = rsc->children;
 725              iter != NULL; iter = iter->next) {
 726 
 727             member = (const pcmk_resource_t *) iter->data;
 728             colocated_rscs = member->cmds->colocated_resources(member, orig_rsc,
 729                                                                colocated_rscs);
 730         }
 731 
 732     } else if (rsc->children != NULL) {
 733         /* This group's members are not colocated, and the group is not cloned,
 734          * so just add the group's own colocations to the list.
 735          */
 736         colocated_rscs = pcmk__colocated_resources(rsc, orig_rsc,
 737                                                    colocated_rscs);
 738     }
 739 
 740     return colocated_rscs;
 741 }
 742 
 743 // Group implementation of pcmk_assignment_methods_t:with_this_colocations()
 744 void
 745 pcmk__with_group_colocations(const pcmk_resource_t *rsc,
     /*  */
 746                              const pcmk_resource_t *orig_rsc, GList **list)
 747 
 748 {
 749     CRM_ASSERT((orig_rsc != NULL) && (list != NULL) && pcmk__is_group(rsc));
 750 
 751     // Ignore empty groups
 752     if (rsc->children == NULL) {
 753         return;
 754     }
 755 
 756     /* "With this" colocations are needed only for the group itself and for its
 757      * last member. (Previous members will chain via the group internal
 758      * colocations.)
 759      */
 760     if ((orig_rsc != rsc) && (orig_rsc != pe__last_group_member(rsc))) {
 761         return;
 762     }
 763 
 764     pcmk__rsc_trace(rsc, "Adding 'with %s' colocations to list for %s",
 765                     rsc->id, orig_rsc->id);
 766 
 767     // Add the group's own colocations
 768     pcmk__add_with_this_list(list, rsc->rsc_cons_lhs, orig_rsc);
 769 
 770     // If cloned, add any relevant colocations with the clone
 771     if (rsc->parent != NULL) {
 772         rsc->parent->cmds->with_this_colocations(rsc->parent, orig_rsc,
 773                                                  list);
 774     }
 775 
 776     if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
 777         // @COMPAT Non-colocated groups are deprecated
 778         return;
 779     }
 780 
 781     // Add explicit colocations with the group's (other) children
 782     for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 783         const pcmk_resource_t *member = iter->data;
 784 
 785         if (member != orig_rsc) {
 786             member->cmds->with_this_colocations(member, orig_rsc, list);
 787         }
 788     }
 789 }
 790 
 791 // Group implementation of pcmk_assignment_methods_t:this_with_colocations()
 792 void
 793 pcmk__group_with_colocations(const pcmk_resource_t *rsc,
     /*  */
 794                              const pcmk_resource_t *orig_rsc, GList **list)
 795 {
 796     const pcmk_resource_t *member = NULL;
 797 
 798     CRM_ASSERT((orig_rsc != NULL) && (list != NULL) && pcmk__is_group(rsc));
 799 
 800     // Ignore empty groups
 801     if (rsc->children == NULL) {
 802         return;
 803     }
 804 
 805     /* "This with" colocations are normally needed only for the group itself and
 806      * for its first member.
 807      */
 808     if ((rsc == orig_rsc)
 809         || (orig_rsc == (const pcmk_resource_t *) rsc->children->data)) {
 810         pcmk__rsc_trace(rsc, "Adding '%s with' colocations to list for %s",
 811                         rsc->id, orig_rsc->id);
 812 
 813         // Add the group's own colocations
 814         pcmk__add_this_with_list(list, rsc->rsc_cons, orig_rsc);
 815 
 816         // If cloned, add any relevant colocations involving the clone
 817         if (rsc->parent != NULL) {
 818             rsc->parent->cmds->this_with_colocations(rsc->parent, orig_rsc,
 819                                                      list);
 820         }
 821 
 822         if (!pe__group_flag_is_set(rsc, pcmk__group_colocated)) {
 823             // @COMPAT Non-colocated groups are deprecated
 824             return;
 825         }
 826 
 827         // Add explicit colocations involving the group's (other) children
 828         for (const GList *iter = rsc->children;
 829              iter != NULL; iter = iter->next) {
 830             member = iter->data;
 831             if (member != orig_rsc) {
 832                 member->cmds->this_with_colocations(member, orig_rsc, list);
 833             }
 834         }
 835         return;
 836     }
 837 
 838     /* Later group members honor the group's colocations indirectly, due to the
 839      * internal group colocations that chain everything from the first member.
 840      * However, if an earlier group member is unmanaged, this chaining will not
 841      * happen, so the group's mandatory colocations must be explicitly added.
 842      */
 843     for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 844         member = iter->data;
 845         if (orig_rsc == member) {
 846             break; // We've seen all earlier members, and none are unmanaged
 847         }
 848 
 849         if (!pcmk_is_set(member->flags, pcmk_rsc_managed)) {
 850             crm_trace("Adding mandatory '%s with' colocations to list for "
 851                       "member %s because earlier member %s is unmanaged",
 852                       rsc->id, orig_rsc->id, member->id);
 853             for (const GList *cons_iter = rsc->rsc_cons; cons_iter != NULL;
 854                  cons_iter = cons_iter->next) {
 855                 const pcmk__colocation_t *colocation = NULL;
 856 
 857                 colocation = (const pcmk__colocation_t *) cons_iter->data;
 858                 if (colocation->score == PCMK_SCORE_INFINITY) {
 859                     pcmk__add_this_with(list, colocation, orig_rsc);
 860                 }
 861             }
 862             // @TODO Add mandatory (or all?) clone constraints if cloned
 863             break;
 864         }
 865     }
 866 }
 867 
 868 /*!
 869  * \internal
 870  * \brief Update nodes with scores of colocated resources' nodes
 871  *
 872  * Given a table of nodes and a resource, update the nodes' scores with the
 873  * scores of the best nodes matching the attribute used for each of the
 874  * resource's relevant colocations.
 875  *
 876  * \param[in,out] source_rsc  Group resource whose node scores to add
 877  * \param[in]     target_rsc  Resource on whose behalf to update \p *nodes
 878  * \param[in]     log_id      Resource ID for logs (if \c NULL, use
 879  *                            \p source_rsc ID)
 880  * \param[in,out] nodes       Nodes to update (set initial contents to \c NULL
 881  *                            to copy allowed nodes from \p source_rsc)
 882  * \param[in]     colocation  Original colocation constraint (used to get
 883  *                            configured primary resource's stickiness, and
 884  *                            to get colocation node attribute; if \c NULL,
 885  *                            <tt>source_rsc</tt>'s own matching node scores will
 886  *                            not be added, and \p *nodes must be \c NULL as
 887  *                            well)
 888  * \param[in]     factor      Incorporate scores multiplied by this factor
 889  * \param[in]     flags       Bitmask of enum pcmk__coloc_select values
 890  *
 891  * \note \c NULL \p target_rsc, \c NULL \p *nodes, \c NULL \p colocation, and
 892  *       the \c pcmk__coloc_select_this_with flag are used together (and only by
 893  *       \c cmp_resources()).
 894  * \note The caller remains responsible for freeing \p *nodes.
 895  * \note This is the group implementation of
 896  *       \c pcmk_assignment_methods_t:add_colocated_node_scores().
 897  */
 898 void
 899 pcmk__group_add_colocated_node_scores(pcmk_resource_t *source_rsc,
     /*  */
 900                                       const pcmk_resource_t *target_rsc,
 901                                       const char *log_id, GHashTable **nodes,
 902                                       const pcmk__colocation_t *colocation,
 903                                       float factor, uint32_t flags)
 904 {
 905     pcmk_resource_t *member = NULL;
 906 
 907     CRM_ASSERT(pcmk__is_group(source_rsc) && (nodes != NULL)
 908                && ((colocation != NULL)
 909                    || ((target_rsc == NULL) && (*nodes == NULL))));
 910 
 911     if (log_id == NULL) {
 912         log_id = source_rsc->id;
 913     }
 914 
 915     // Avoid infinite recursion
 916     if (pcmk_is_set(source_rsc->flags, pcmk_rsc_updating_nodes)) {
 917         pcmk__rsc_info(source_rsc, "%s: Breaking dependency loop at %s",
 918                        log_id, source_rsc->id);
 919         return;
 920     }
 921     pcmk__set_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);
 922 
 923     // Ignore empty groups (only possible with schema validation disabled)
 924     if (source_rsc->children == NULL) {
 925         return;
 926     }
 927 
 928     /* Refer the operation to the first or last member as appropriate.
 929      *
 930      * cmp_resources() is the only caller that passes a NULL nodes table,
 931      * and is also the only caller using pcmk__coloc_select_this_with.
 932      * For "this with" colocations, the last member will recursively incorporate
 933      * all the other members' "this with" colocations via the internal group
 934      * colocations (and via the first member, the group's own colocations).
 935      *
 936      * For "with this" colocations, the first member works similarly.
 937      */
 938     if (*nodes == NULL) {
 939         member = pe__last_group_member(source_rsc);
 940     } else {
 941         member = source_rsc->children->data;
 942     }
 943     pcmk__rsc_trace(source_rsc, "%s: Merging scores from group %s using member %s "
 944                     "(at %.6f)", log_id, source_rsc->id, member->id, factor);
 945     member->cmds->add_colocated_node_scores(member, target_rsc, log_id, nodes,
 946                                             colocation, factor, flags);
 947     pcmk__clear_rsc_flags(source_rsc, pcmk_rsc_updating_nodes);
 948 }
 949 
 950 // Group implementation of pcmk_assignment_methods_t:add_utilization()
 951 void
 952 pcmk__group_add_utilization(const pcmk_resource_t *rsc,
     /*  */
 953                             const pcmk_resource_t *orig_rsc, GList *all_rscs,
 954                             GHashTable *utilization)
 955 {
 956     pcmk_resource_t *member = NULL;
 957 
 958     CRM_ASSERT((orig_rsc != NULL) && (utilization != NULL)
 959                && pcmk__is_group(rsc));
 960 
 961     if (!pcmk_is_set(rsc->flags, pcmk_rsc_unassigned)) {
 962         return;
 963     }
 964 
 965     pcmk__rsc_trace(orig_rsc, "%s: Adding group %s as colocated utilization",
 966                     orig_rsc->id, rsc->id);
 967     if (pe__group_flag_is_set(rsc, pcmk__group_colocated)
 968         || pcmk__is_clone(rsc->parent)) {
 969         // Every group member will be on same node, so sum all members
 970         for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 971             member = (pcmk_resource_t *) iter->data;
 972 
 973             if (pcmk_is_set(member->flags, pcmk_rsc_unassigned)
 974                 && (g_list_find(all_rscs, member) == NULL)) {
 975                 member->cmds->add_utilization(member, orig_rsc, all_rscs,
 976                                               utilization);
 977             }
 978         }
 979 
 980     } else if (rsc->children != NULL) {
 981         // Just add first member's utilization
 982         member = (pcmk_resource_t *) rsc->children->data;
 983         if ((member != NULL)
 984             && pcmk_is_set(member->flags, pcmk_rsc_unassigned)
 985             && (g_list_find(all_rscs, member) == NULL)) {
 986 
 987             member->cmds->add_utilization(member, orig_rsc, all_rscs,
 988                                           utilization);
 989         }
 990     }
 991 }
 992 
 993 void
 994 pcmk__group_shutdown_lock(pcmk_resource_t *rsc)
     /*  */
 995 {
 996     CRM_ASSERT(pcmk__is_group(rsc));
 997 
 998     for (GList *iter = rsc->children; iter != NULL; iter = iter->next) {
 999         pcmk_resource_t *member = (pcmk_resource_t *) iter->data;
1000 
1001         member->cmds->shutdown_lock(member);
1002     }
1003 }

/* */