root/lib/pacemaker/pcmk_sched_bundle.c

/* */

DEFINITIONS

1. is_bundle_node
2. pcmk__bundle_allocate
3. pcmk__bundle_create_actions
4. pcmk__bundle_internal_constraints
5. compatible_replica_for_node
6. compatible_replica
7. copies_per_node
8. pcmk__bundle_apply_coloc_score
9. pcmk__with_bundle_colocations
10. pcmk__bundle_with_colocations
11. pcmk__bundle_action_flags
12. pcmk__get_rsc_in_container
13. pcmk__bundle_rsc_location
14. pcmk__bundle_expand
15. pcmk__bundle_create_probe
16. pcmk__output_bundle_actions
17. pcmk__bundle_add_utilization
18. pcmk__bundle_shutdown_lock

   1 /*
   2  * Copyright 2004-2023 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdbool.h>
  13 
  14 #include <crm/msg_xml.h>
  15 #include <pacemaker-internal.h>
  16 
  17 #include "libpacemaker_private.h"
  18 
  19 #define PE__VARIANT_BUNDLE 1
  20 #include <lib/pengine/variant.h>
  21 
  22 static bool
  23 is_bundle_node(pe__bundle_variant_data_t *data, pe_node_t *node)
     /*  */
  24 {
  25     for (GList *gIter = data->replicas; gIter != NULL; gIter = gIter->next) {
  26         pe__bundle_replica_t *replica = gIter->data;
  27 
  28         if (node->details == replica->node->details) {
  29             return TRUE;
  30         }
  31     }
  32     return FALSE;
  33 }
  34 
  35 /*!
  36  * \internal
  37  * \brief Assign a bundle resource to a node
  38  *
  39  * \param[in,out] rsc     Resource to assign to a node
  40  * \param[in]     prefer  Node to prefer, if all else is equal
  41  *
  42  * \return Node that \p rsc is assigned to, if assigned entirely to one node
  43  */
  44 pe_node_t *
  45 pcmk__bundle_allocate(pe_resource_t *rsc, const pe_node_t *prefer)
     /*  */
  46 {
  47     GList *containers = NULL;
  48     pe__bundle_variant_data_t *bundle_data = NULL;
  49 
  50     CRM_CHECK(rsc != NULL, return NULL);
  51 
  52     get_bundle_variant_data(bundle_data, rsc);
  53 
  54     pe__set_resource_flags(rsc, pe_rsc_allocating);
  55     containers = pe__bundle_containers(rsc);
  56 
  57     pe__show_node_weights(!pcmk_is_set(rsc->cluster->flags, pe_flag_show_scores),
  58                           rsc, __func__, rsc->allowed_nodes, rsc->cluster);
  59 
  60     containers = g_list_sort(containers, pcmk__cmp_instance);
  61     pcmk__assign_instances(rsc, containers, bundle_data->nreplicas,
  62                            bundle_data->nreplicas_per_host);
  63     g_list_free(containers);
  64 
  65     for (GList *gIter = bundle_data->replicas; gIter != NULL;
  66          gIter = gIter->next) {
  67         pe__bundle_replica_t *replica = gIter->data;
  68         pe_node_t *container_host = NULL;
  69 
  70         CRM_ASSERT(replica);
  71         if (replica->ip) {
  72             pe_rsc_trace(rsc, "Allocating bundle %s IP %s",
  73                          rsc->id, replica->ip->id);
  74             replica->ip->cmds->assign(replica->ip, prefer);
  75         }
  76 
  77         container_host = replica->container->allocated_to;
  78         if (replica->remote && pe__is_guest_or_remote_node(container_host)) {
  79             /* We need 'nested' connection resources to be on the same
  80              * host because pacemaker-remoted only supports a single
  81              * active connection
  82              */
  83             pcmk__new_colocation("child-remote-with-docker-remote", NULL,
  84                                  INFINITY, replica->remote,
  85                                  container_host->details->remote_rsc, NULL,
  86                                  NULL, true, rsc->cluster);
  87         }
  88 
  89         if (replica->remote) {
  90             pe_rsc_trace(rsc, "Allocating bundle %s connection %s",
  91                          rsc->id, replica->remote->id);
  92             replica->remote->cmds->assign(replica->remote, prefer);
  93         }
  94 
  95         // Explicitly allocate replicas' children before bundle child
  96         if (replica->child) {
  97             pe_node_t *node = NULL;
  98             GHashTableIter iter;
  99 
 100             g_hash_table_iter_init(&iter, replica->child->allowed_nodes);
 101             while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
 102                 if (node->details != replica->node->details) {
 103                     node->weight = -INFINITY;
 104                 } else if (!pcmk__threshold_reached(replica->child, node,
 105                                                     NULL)) {
 106                     node->weight = INFINITY;
 107                 }
 108             }
 109 
 110             pe__set_resource_flags(replica->child->parent, pe_rsc_allocating);
 111             pe_rsc_trace(rsc, "Allocating bundle %s replica child %s",
 112                          rsc->id, replica->child->id);
 113             replica->child->cmds->assign(replica->child, replica->node);
 114             pe__clear_resource_flags(replica->child->parent,
 115                                        pe_rsc_allocating);
 116         }
 117     }
 118 
 119     if (bundle_data->child) {
 120         pe_node_t *node = NULL;
 121         GHashTableIter iter;
 122         g_hash_table_iter_init(&iter, bundle_data->child->allowed_nodes);
 123         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
 124             if (is_bundle_node(bundle_data, node)) {
 125                 node->weight = 0;
 126             } else {
 127                 node->weight = -INFINITY;
 128             }
 129         }
 130         pe_rsc_trace(rsc, "Allocating bundle %s child %s",
 131                      rsc->id, bundle_data->child->id);
 132         bundle_data->child->cmds->assign(bundle_data->child, prefer);
 133     }
 134 
 135     pe__clear_resource_flags(rsc, pe_rsc_allocating|pe_rsc_provisional);
 136     return NULL;
 137 }
 138 
 139 
 140 void
 141 pcmk__bundle_create_actions(pe_resource_t *rsc)
     /*  */
 142 {
 143     pe_action_t *action = NULL;
 144     GList *containers = NULL;
 145     pe__bundle_variant_data_t *bundle_data = NULL;
 146 
 147     CRM_CHECK(rsc != NULL, return);
 148 
 149     containers = pe__bundle_containers(rsc);
 150     get_bundle_variant_data(bundle_data, rsc);
 151     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 152          gIter = gIter->next) {
 153         pe__bundle_replica_t *replica = gIter->data;
 154 
 155         CRM_ASSERT(replica);
 156         if (replica->ip) {
 157             replica->ip->cmds->create_actions(replica->ip);
 158         }
 159         if (replica->container) {
 160             replica->container->cmds->create_actions(replica->container);
 161         }
 162         if (replica->remote) {
 163             replica->remote->cmds->create_actions(replica->remote);
 164         }
 165     }
 166 
 167     pcmk__create_instance_actions(rsc, containers);
 168 
 169     if (bundle_data->child) {
 170         bundle_data->child->cmds->create_actions(bundle_data->child);
 171 
 172         if (pcmk_is_set(bundle_data->child->flags, pe_rsc_promotable)) {
 173             /* promote */
 174             pe__new_rsc_pseudo_action(rsc, RSC_PROMOTE, true, true);
 175             action = pe__new_rsc_pseudo_action(rsc, RSC_PROMOTED, true, true);
 176             action->priority = INFINITY;
 177 
 178             /* demote */
 179             pe__new_rsc_pseudo_action(rsc, RSC_DEMOTE, true, true);
 180             action = pe__new_rsc_pseudo_action(rsc, RSC_DEMOTED, true, true);
 181             action->priority = INFINITY;
 182         }
 183     }
 184 
 185     g_list_free(containers);
 186 }
 187 
 188 void
 189 pcmk__bundle_internal_constraints(pe_resource_t *rsc)
     /*  */
 190 {
 191     pe__bundle_variant_data_t *bundle_data = NULL;
 192 
 193     CRM_CHECK(rsc != NULL, return);
 194 
 195     get_bundle_variant_data(bundle_data, rsc);
 196 
 197     if (bundle_data->child) {
 198         pcmk__order_resource_actions(rsc, RSC_START, bundle_data->child,
 199                                      RSC_START, pe_order_implies_first_printed);
 200         pcmk__order_resource_actions(rsc, RSC_STOP, bundle_data->child,
 201                                      RSC_STOP, pe_order_implies_first_printed);
 202 
 203         if (bundle_data->child->children) {
 204             pcmk__order_resource_actions(bundle_data->child, RSC_STARTED, rsc,
 205                                          RSC_STARTED,
 206                                          pe_order_implies_then_printed);
 207             pcmk__order_resource_actions(bundle_data->child, RSC_STOPPED, rsc,
 208                                          RSC_STOPPED,
 209                                          pe_order_implies_then_printed);
 210         } else {
 211             pcmk__order_resource_actions(bundle_data->child, RSC_START, rsc,
 212                                          RSC_STARTED,
 213                                          pe_order_implies_then_printed);
 214             pcmk__order_resource_actions(bundle_data->child, RSC_STOP, rsc,
 215                                          RSC_STOPPED,
 216                                          pe_order_implies_then_printed);
 217         }
 218     }
 219 
 220     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 221          gIter = gIter->next) {
 222         pe__bundle_replica_t *replica = gIter->data;
 223 
 224         CRM_ASSERT(replica);
 225         CRM_ASSERT(replica->container);
 226 
 227         replica->container->cmds->internal_constraints(replica->container);
 228 
 229         pcmk__order_starts(rsc, replica->container,
 230                            pe_order_runnable_left|pe_order_implies_first_printed);
 231 
 232         if (replica->child) {
 233             pcmk__order_stops(rsc, replica->child,
 234                               pe_order_implies_first_printed);
 235         }
 236         pcmk__order_stops(rsc, replica->container,
 237                           pe_order_implies_first_printed);
 238         pcmk__order_resource_actions(replica->container, RSC_START, rsc,
 239                                      RSC_STARTED,
 240                                      pe_order_implies_then_printed);
 241         pcmk__order_resource_actions(replica->container, RSC_STOP, rsc,
 242                                      RSC_STOPPED,
 243                                      pe_order_implies_then_printed);
 244 
 245         if (replica->ip) {
 246             replica->ip->cmds->internal_constraints(replica->ip);
 247 
 248             // Start IP then container
 249             pcmk__order_starts(replica->ip, replica->container,
 250                                pe_order_runnable_left|pe_order_preserve);
 251             pcmk__order_stops(replica->container, replica->ip,
 252                               pe_order_implies_first|pe_order_preserve);
 253 
 254             pcmk__new_colocation("ip-with-docker", NULL, INFINITY, replica->ip,
 255                                  replica->container, NULL, NULL, true,
 256                                  rsc->cluster);
 257         }
 258 
 259         if (replica->remote) {
 260             /* This handles ordering and colocating remote relative to container
 261              * (via "resource-with-container"). Since IP is also ordered and
 262              * colocated relative to the container, we don't need to do anything
 263              * explicit here with IP.
 264              */
 265             replica->remote->cmds->internal_constraints(replica->remote);
 266         }
 267 
 268         if (replica->child) {
 269             CRM_ASSERT(replica->remote);
 270 
 271             // "Start remote then child" is implicit in scheduler's remote logic
 272         }
 273 
 274     }
 275 
 276     if (bundle_data->child) {
 277         bundle_data->child->cmds->internal_constraints(bundle_data->child);
 278         if (pcmk_is_set(bundle_data->child->flags, pe_rsc_promotable)) {
 279             pcmk__promotable_restart_ordering(rsc);
 280 
 281             /* child demoted before global demoted */
 282             pcmk__order_resource_actions(bundle_data->child, RSC_DEMOTED, rsc,
 283                                          RSC_DEMOTED,
 284                                          pe_order_implies_then_printed);
 285 
 286             /* global demote before child demote */
 287             pcmk__order_resource_actions(rsc, RSC_DEMOTE, bundle_data->child,
 288                                          RSC_DEMOTE,
 289                                          pe_order_implies_first_printed);
 290 
 291             /* child promoted before global promoted */
 292             pcmk__order_resource_actions(bundle_data->child, RSC_PROMOTED, rsc,
 293                                          RSC_PROMOTED,
 294                                          pe_order_implies_then_printed);
 295 
 296             /* global promote before child promote */
 297             pcmk__order_resource_actions(rsc, RSC_PROMOTE, bundle_data->child,
 298                                          RSC_PROMOTE,
 299                                          pe_order_implies_first_printed);
 300         }
 301     }
 302 }
 303 
 304 static pe_resource_t *
 305 compatible_replica_for_node(const pe_resource_t *rsc_lh,
     /*  */
 306                             const pe_node_t *candidate,
 307                             const pe_resource_t *rsc, enum rsc_role_e filter,
 308                             gboolean current)
 309 {
 310     pe__bundle_variant_data_t *bundle_data = NULL;
 311 
 312     CRM_CHECK(candidate != NULL, return NULL);
 313     get_bundle_variant_data(bundle_data, rsc);
 314 
 315     crm_trace("Looking for compatible child from %s for %s on %s",
 316               rsc_lh->id, rsc->id, pe__node_name(candidate));
 317 
 318     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 319          gIter = gIter->next) {
 320         pe__bundle_replica_t *replica = gIter->data;
 321 
 322         if (pcmk__instance_matches(replica->container, candidate, filter,
 323                                    current)) {
 324             crm_trace("Pairing %s with %s on %s",
 325                       rsc_lh->id, replica->container->id,
 326                       pe__node_name(candidate));
 327             return replica->container;
 328         }
 329     }
 330 
 331     crm_trace("Can't pair %s with %s", rsc_lh->id, rsc->id);
 332     return NULL;
 333 }
 334 
 335 static pe_resource_t *
 336 compatible_replica(const pe_resource_t *rsc_lh, const pe_resource_t *rsc,
     /*  */
 337                    enum rsc_role_e filter, gboolean current,
 338                    pe_working_set_t *data_set)
 339 {
 340     GList *scratch = NULL;
 341     pe_resource_t *pair = NULL;
 342     pe_node_t *active_node_lh = NULL;
 343 
 344     active_node_lh = rsc_lh->fns->location(rsc_lh, NULL, current);
 345     if (active_node_lh) {
 346         return compatible_replica_for_node(rsc_lh, active_node_lh, rsc, filter,
 347                                            current);
 348     }
 349 
 350     scratch = g_hash_table_get_values(rsc_lh->allowed_nodes);
 351     scratch = pcmk__sort_nodes(scratch, NULL);
 352 
 353     for (GList *gIter = scratch; gIter != NULL; gIter = gIter->next) {
 354         pe_node_t *node = (pe_node_t *) gIter->data;
 355 
 356         pair = compatible_replica_for_node(rsc_lh, node, rsc, filter, current);
 357         if (pair) {
 358             goto done;
 359         }
 360     }
 361 
 362     pe_rsc_debug(rsc, "Can't pair %s with %s", rsc_lh->id, (rsc? rsc->id : "none"));
 363   done:
 364     g_list_free(scratch);
 365     return pair;
 366 }
 367 
 368 int copies_per_node(pe_resource_t * rsc) 
     /*  */
 369 {
 370     /* Strictly speaking, there should be a 'copies_per_node' addition
 371      * to the resource function table and each case would be a
 372      * function.  However that would be serious overkill to return an
 373      * int.  In fact, it seems to me that both function tables
 374      * could/should be replaced by resources.{c,h} full of
 375      * rsc_{some_operation} functions containing a switch as below
 376      * which calls out to functions named {variant}_{some_operation}
 377      * as needed.
 378      */
 379     switch(rsc->variant) {
 380         case pe_unknown:
 381             return 0;
 382         case pe_native:
 383         case pe_group:
 384             return 1;
 385         case pe_clone:
 386             {
 387                 const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);
 388 
 389                 if (max_clones_node == NULL) {
 390                     return 1;
 391 
 392                 } else {
 393                     int max_i;
 394 
 395                     pcmk__scan_min_int(max_clones_node, &max_i, 0);
 396                     return max_i;
 397                 }
 398             }
 399         case pe_container:
 400             {
 401                 pe__bundle_variant_data_t *data = NULL;
 402                 get_bundle_variant_data(data, rsc);
 403                 return data->nreplicas_per_host;
 404             }
 405     }
 406     return 0;
 407 }
 408 
 409 /*!
 410  * \internal
 411  * \brief Apply a colocation's score to node weights or resource priority
 412  *
 413  * Given a colocation constraint, apply its score to the dependent's
 414  * allowed node weights (if we are still placing resources) or priority (if
 415  * we are choosing promotable clone instance roles).
 416  *
 417  * \param[in,out] dependent      Dependent resource in colocation
 418  * \param[in]     primary        Primary resource in colocation
 419  * \param[in]     colocation     Colocation constraint to apply
 420  * \param[in]     for_dependent  true if called on behalf of dependent
 421  */
 422 void
 423 pcmk__bundle_apply_coloc_score(pe_resource_t *dependent,
     /*  */
 424                                const pe_resource_t *primary,
 425                                const pcmk__colocation_t *colocation,
 426                                bool for_dependent)
 427 {
 428     GList *allocated_primaries = NULL;
 429     pe__bundle_variant_data_t *bundle_data = NULL;
 430 
 431     /* This should never be called for the bundle itself as a dependent.
 432      * Instead, we add its colocation constraints to its replicas and call the
 433      * apply_coloc_score() for the replicas as dependents.
 434      */
 435     CRM_ASSERT(!for_dependent);
 436 
 437     CRM_CHECK((colocation != NULL) && (dependent != NULL) && (primary != NULL),
 438               return);
 439     CRM_ASSERT(dependent->variant == pe_native);
 440 
 441     if (pcmk_is_set(primary->flags, pe_rsc_provisional)) {
 442         pe_rsc_trace(primary, "%s is still provisional", primary->id);
 443         return;
 444 
 445     } else if (colocation->dependent->variant > pe_group) {
 446         pe_resource_t *primary_replica = compatible_replica(dependent, primary,
 447                                                             RSC_ROLE_UNKNOWN,
 448                                                             FALSE,
 449                                                             dependent->cluster);
 450 
 451         if (primary_replica) {
 452             pe_rsc_debug(primary, "Pairing %s with %s",
 453                          dependent->id, primary_replica->id);
 454             dependent->cmds->apply_coloc_score(dependent, primary_replica,
 455                                                colocation, true);
 456 
 457         } else if (colocation->score >= INFINITY) {
 458             crm_notice("Cannot pair %s with instance of %s",
 459                        dependent->id, primary->id);
 460             pcmk__assign_resource(dependent, NULL, true);
 461 
 462         } else {
 463             pe_rsc_debug(primary, "Cannot pair %s with instance of %s",
 464                          dependent->id, primary->id);
 465         }
 466 
 467         return;
 468     }
 469 
 470     get_bundle_variant_data(bundle_data, primary);
 471     pe_rsc_trace(primary, "Processing constraint %s: %s -> %s %d",
 472                  colocation->id, dependent->id, primary->id, colocation->score);
 473 
 474     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 475          gIter = gIter->next) {
 476         pe__bundle_replica_t *replica = gIter->data;
 477 
 478         if (colocation->score < INFINITY) {
 479             replica->container->cmds->apply_coloc_score(dependent,
 480                                                         replica->container,
 481                                                         colocation, false);
 482 
 483         } else {
 484             pe_node_t *chosen = replica->container->fns->location(replica->container,
 485                                                                   NULL, FALSE);
 486 
 487             if ((chosen == NULL)
 488                 || is_set_recursive(replica->container, pe_rsc_block, TRUE)) {
 489                 continue;
 490             }
 491             if ((colocation->primary_role >= RSC_ROLE_PROMOTED)
 492                 && (replica->child == NULL)) {
 493                 continue;
 494             }
 495             if ((colocation->primary_role >= RSC_ROLE_PROMOTED)
 496                 && (replica->child->next_role < RSC_ROLE_PROMOTED)) {
 497                 continue;
 498             }
 499 
 500             pe_rsc_trace(primary, "Allowing %s: %s %d",
 501                          colocation->id, pe__node_name(chosen), chosen->weight);
 502             allocated_primaries = g_list_prepend(allocated_primaries, chosen);
 503         }
 504     }
 505 
 506     if (colocation->score >= INFINITY) {
 507         node_list_exclude(dependent->allowed_nodes, allocated_primaries, FALSE);
 508     }
 509     g_list_free(allocated_primaries);
 510 }
 511 
 512 // Bundle implementation of resource_alloc_functions_t:with_this_colocations()
 513 void
 514 pcmk__with_bundle_colocations(const pe_resource_t *rsc,
     /*  */
 515                               const pe_resource_t *orig_rsc, GList **list)
 516 {
 517     CRM_CHECK((rsc != NULL) && (rsc->variant == pe_container)
 518               && (orig_rsc != NULL) && (list != NULL),
 519               return);
 520 
 521     if (rsc == orig_rsc) { // Colocations are wanted for bundle itself
 522         pcmk__add_with_this_list(list, rsc->rsc_cons_lhs);
 523 
 524     // Only the bundle replicas' containers get the bundle's constraints
 525     } else if (pcmk_is_set(orig_rsc->flags, pe_rsc_replica_container)) {
 526         pcmk__add_collective_constraints(list, orig_rsc, rsc, true);
 527     }
 528 }
 529 
 530 // Bundle implementation of resource_alloc_functions_t:this_with_colocations()
 531 void
 532 pcmk__bundle_with_colocations(const pe_resource_t *rsc,
     /*  */
 533                               const pe_resource_t *orig_rsc, GList **list)
 534 {
 535     CRM_CHECK((rsc != NULL) && (rsc->variant == pe_container)
 536               && (orig_rsc != NULL) && (list != NULL),
 537               return);
 538 
 539     if (rsc == orig_rsc) { // Colocations are wanted for bundle itself
 540         pcmk__add_this_with_list(list, rsc->rsc_cons);
 541 
 542     // Only the bundle replicas' containers get the bundle's constraints
 543     } else if (pcmk_is_set(orig_rsc->flags, pe_rsc_replica_container)) {
 544         pcmk__add_collective_constraints(list, orig_rsc, rsc, false);
 545     }
 546 }
 547 
 548 enum pe_action_flags
 549 pcmk__bundle_action_flags(pe_action_t *action, const pe_node_t *node)
     /*  */
 550 {
 551     GList *containers = NULL;
 552     enum pe_action_flags flags = 0;
 553     pe__bundle_variant_data_t *data = NULL;
 554 
 555     get_bundle_variant_data(data, action->rsc);
 556     if(data->child) {
 557         enum action_tasks task = get_complex_task(data->child, action->task);
 558         switch(task) {
 559             case no_action:
 560             case action_notify:
 561             case action_notified:
 562             case action_promote:
 563             case action_promoted:
 564             case action_demote:
 565             case action_demoted:
 566                 return pcmk__collective_action_flags(action,
 567                                                      data->child->children,
 568                                                      node);
 569             default:
 570                 break;
 571         }
 572     }
 573 
 574     containers = pe__bundle_containers(action->rsc);
 575     flags = pcmk__collective_action_flags(action, containers, node);
 576     g_list_free(containers);
 577     return flags;
 578 }
 579 
 580 /*!
 581  * \internal
 582  * \brief Get containerized resource corresponding to a given bundle container
 583  *
 584  * \param[in] instance  Collective instance that might be a bundle container
 585  *
 586  * \return Bundled resource instance inside \p instance if it is a bundle
 587  *         container instance, otherwise NULL
 588  */
 589 const pe_resource_t *
 590 pcmk__get_rsc_in_container(const pe_resource_t *instance)
     /*  */
 591 {
 592     const pe__bundle_variant_data_t *data = NULL;
 593     const pe_resource_t *top = pe__const_top_resource(instance, true);
 594 
 595     if ((top == NULL) || (top->variant != pe_container)) {
 596         return NULL;
 597     }
 598     get_bundle_variant_data(data, top);
 599 
 600     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 601         const pe__bundle_replica_t *replica = iter->data;
 602 
 603         if (instance == replica->container) {
 604             return replica->child;
 605         }
 606     }
 607     return NULL;
 608 }
 609 
 610 void
 611 pcmk__bundle_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
     /*  */
 612 {
 613     pe__bundle_variant_data_t *bundle_data = NULL;
 614     get_bundle_variant_data(bundle_data, rsc);
 615 
 616     pcmk__apply_location(rsc, constraint);
 617 
 618     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 619          gIter = gIter->next) {
 620         pe__bundle_replica_t *replica = gIter->data;
 621 
 622         if (replica->container) {
 623             replica->container->cmds->apply_location(replica->container,
 624                                                      constraint);
 625         }
 626         if (replica->ip) {
 627             replica->ip->cmds->apply_location(replica->ip, constraint);
 628         }
 629     }
 630 
 631     if (bundle_data->child
 632         && ((constraint->role_filter == RSC_ROLE_UNPROMOTED)
 633             || (constraint->role_filter == RSC_ROLE_PROMOTED))) {
 634         bundle_data->child->cmds->apply_location(bundle_data->child,
 635                                                  constraint);
 636         bundle_data->child->rsc_location = g_list_prepend(bundle_data->child->rsc_location,
 637                                                           constraint);
 638     }
 639 }
 640 
 641 /*!
 642  * \internal
 643  * \brief Add a resource's actions to the transition graph
 644  *
 645  * \param[in,out] rsc  Resource whose actions should be added
 646  */
 647 void
 648 pcmk__bundle_expand(pe_resource_t *rsc)
     /*  */
 649 {
 650     pe__bundle_variant_data_t *bundle_data = NULL;
 651 
 652     CRM_CHECK(rsc != NULL, return);
 653 
 654     get_bundle_variant_data(bundle_data, rsc);
 655 
 656     if (bundle_data->child) {
 657         bundle_data->child->cmds->add_actions_to_graph(bundle_data->child);
 658     }
 659 
 660     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 661          gIter = gIter->next) {
 662         pe__bundle_replica_t *replica = gIter->data;
 663 
 664         CRM_ASSERT(replica);
 665         if (replica->remote && replica->container
 666             && pe__bundle_needs_remote_name(replica->remote)) {
 667 
 668             /* REMOTE_CONTAINER_HACK: Allow remote nodes to run containers that
 669              * run pacemaker-remoted inside, without needing a separate IP for
 670              * the container. This is done by configuring the inner remote's
 671              * connection host as the magic string "#uname", then
 672              * replacing it with the underlying host when needed.
 673              */
 674             xmlNode *nvpair = get_xpath_object("//nvpair[@name='" XML_RSC_ATTR_REMOTE_RA_ADDR "']",
 675                                                replica->remote->xml, LOG_ERR);
 676             const char *calculated_addr = NULL;
 677 
 678             // Replace the value in replica->remote->xml (if appropriate)
 679             calculated_addr = pe__add_bundle_remote_name(replica->remote,
 680                                                          rsc->cluster,
 681                                                          nvpair, "value");
 682             if (calculated_addr) {
 683                 /* Since this is for the bundle as a resource, and not any
 684                  * particular action, replace the value in the default
 685                  * parameters (not evaluated for node). create_graph_action()
 686                  * will grab it from there to replace it in node-evaluated
 687                  * parameters.
 688                  */
 689                 GHashTable *params = pe_rsc_params(replica->remote,
 690                                                    NULL, rsc->cluster);
 691 
 692                 g_hash_table_replace(params,
 693                                      strdup(XML_RSC_ATTR_REMOTE_RA_ADDR),
 694                                      strdup(calculated_addr));
 695             } else {
 696                 /* The only way to get here is if the remote connection is
 697                  * neither currently running nor scheduled to run. That means we
 698                  * won't be doing any operations that require addr (only start
 699                  * requires it; we additionally use it to compare digests when
 700                  * unpacking status, promote, and migrate_from history, but
 701                  * that's already happened by this point).
 702                  */
 703                 crm_info("Unable to determine address for bundle %s remote connection",
 704                          rsc->id);
 705             }
 706         }
 707         if (replica->ip) {
 708             replica->ip->cmds->add_actions_to_graph(replica->ip);
 709         }
 710         if (replica->container) {
 711             replica->container->cmds->add_actions_to_graph(replica->container);
 712         }
 713         if (replica->remote) {
 714             replica->remote->cmds->add_actions_to_graph(replica->remote);
 715         }
 716     }
 717 }
 718 
 719 /*!
 720  * \internal
 721  *
 722  * \brief Schedule any probes needed for a resource on a node
 723  *
 724  * \param[in,out] rsc   Resource to create probe for
 725  * \param[in,out] node  Node to create probe on
 726  *
 727  * \return true if any probe was created, otherwise false
 728  */
 729 bool
 730 pcmk__bundle_create_probe(pe_resource_t *rsc, pe_node_t *node)
     /*  */
 731 {
 732     bool any_created = false;
 733     pe__bundle_variant_data_t *bundle_data = NULL;
 734 
 735     CRM_CHECK(rsc != NULL, return false);
 736 
 737     get_bundle_variant_data(bundle_data, rsc);
 738     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 739          gIter = gIter->next) {
 740         pe__bundle_replica_t *replica = gIter->data;
 741 
 742         CRM_ASSERT(replica);
 743         if ((replica->ip != NULL)
 744             && replica->ip->cmds->create_probe(replica->ip, node)) {
 745             any_created = true;
 746         }
 747         if ((replica->child != NULL) && (node->details == replica->node->details)
 748             && replica->child->cmds->create_probe(replica->child, node)) {
 749             any_created = true;
 750         }
 751         if ((replica->container != NULL)
 752             && replica->container->cmds->create_probe(replica->container,
 753                                                       node)) {
 754             any_created = true;
 755 
 756             /* If we're limited to one replica per host (due to
 757              * the lack of an IP range probably), then we don't
 758              * want any of our peer containers starting until
 759              * we've established that no other copies are already
 760              * running.
 761              *
 762              * Partly this is to ensure that nreplicas_per_host is
 763              * observed, but also to ensure that the containers
 764              * don't fail to start because the necessary port
 765              * mappings (which won't include an IP for uniqueness)
 766              * are already taken
 767              */
 768 
 769             for (GList *tIter = bundle_data->replicas;
 770                  tIter && (bundle_data->nreplicas_per_host == 1);
 771                  tIter = tIter->next) {
 772                 pe__bundle_replica_t *other = tIter->data;
 773 
 774                 if ((other != replica) && (other != NULL)
 775                     && (other->container != NULL)) {
 776 
 777                     pcmk__new_ordering(replica->container,
 778                                        pcmk__op_key(replica->container->id, RSC_STATUS, 0),
 779                                        NULL, other->container,
 780                                        pcmk__op_key(other->container->id, RSC_START, 0),
 781                                        NULL,
 782                                        pe_order_optional|pe_order_same_node,
 783                                        rsc->cluster);
 784                 }
 785             }
 786         }
 787         if ((replica->container != NULL) && (replica->remote != NULL)
 788             && replica->remote->cmds->create_probe(replica->remote, node)) {
 789 
 790             /* Do not probe the remote resource until we know where the
 791              * container is running. This is required for REMOTE_CONTAINER_HACK
 792              * to correctly probe remote resources.
 793              */
 794             char *probe_uuid = pcmk__op_key(replica->remote->id, RSC_STATUS,
 795                                                0);
 796             pe_action_t *probe = find_first_action(replica->remote->actions,
 797                                                    probe_uuid, NULL, node);
 798 
 799             free(probe_uuid);
 800             if (probe != NULL) {
 801                 any_created = true;
 802                 crm_trace("Ordering %s probe on %s",
 803                           replica->remote->id, pe__node_name(node));
 804                 pcmk__new_ordering(replica->container,
 805                                    pcmk__op_key(replica->container->id, RSC_START, 0),
 806                                    NULL, replica->remote, NULL, probe,
 807                                    pe_order_probe, rsc->cluster);
 808             }
 809         }
 810     }
 811     return any_created;
 812 }
 813 
 814 void
 815 pcmk__output_bundle_actions(pe_resource_t *rsc)
     /*  */
 816 {
 817     pe__bundle_variant_data_t *bundle_data = NULL;
 818 
 819     CRM_CHECK(rsc != NULL, return);
 820 
 821     get_bundle_variant_data(bundle_data, rsc);
 822     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 823          gIter = gIter->next) {
 824         pe__bundle_replica_t *replica = gIter->data;
 825 
 826         CRM_ASSERT(replica);
 827         if (replica->ip != NULL) {
 828             replica->ip->cmds->output_actions(replica->ip);
 829         }
 830         if (replica->container != NULL) {
 831             replica->container->cmds->output_actions(replica->container);
 832         }
 833         if (replica->remote != NULL) {
 834             replica->remote->cmds->output_actions(replica->remote);
 835         }
 836         if (replica->child != NULL) {
 837             replica->child->cmds->output_actions(replica->child);
 838         }
 839     }
 840 }
 841 
 842 // Bundle implementation of resource_alloc_functions_t:add_utilization()
 843 void
 844 pcmk__bundle_add_utilization(const pe_resource_t *rsc,
     /*  */
 845                              const pe_resource_t *orig_rsc, GList *all_rscs,
 846                              GHashTable *utilization)
 847 {
 848     pe__bundle_variant_data_t *bundle_data = NULL;
 849     pe__bundle_replica_t *replica = NULL;
 850 
 851     if (!pcmk_is_set(rsc->flags, pe_rsc_provisional)) {
 852         return;
 853     }
 854 
 855     get_bundle_variant_data(bundle_data, rsc);
 856     if (bundle_data->replicas == NULL) {
 857         return;
 858     }
 859 
 860     /* All bundle replicas are identical, so using the utilization of the first
 861      * is sufficient for any. Only the implicit container resource can have
 862      * utilization values.
 863      */
 864     replica = (pe__bundle_replica_t *) bundle_data->replicas->data;
 865     if (replica->container != NULL) {
 866         replica->container->cmds->add_utilization(replica->container, orig_rsc,
 867                                                   all_rscs, utilization);
 868     }
 869 }
 870 
 871 // Bundle implementation of resource_alloc_functions_t:shutdown_lock()
 872 void
 873 pcmk__bundle_shutdown_lock(pe_resource_t *rsc)
     /*  */
 874 {
 875     return; // Bundles currently don't support shutdown locks
 876 }

/* */