lib/pengine/bundle.c

/* */
This source file includes following definitions.
pe__bundle_max
pe__bundled_resource
pe__get_rsc_in_container
pe__node_is_bundle_instance
pe__first_container
pe__foreach_bundle_replica
pe__foreach_const_bundle_replica
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
container_agent_str
create_container_resource
disallow_node
create_remote_resource
create_replica_resources
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
get_unmanaged_str
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
pe__bundle_containers
pe__bundle_active_node
pe__bundle_max_per_node
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/common/xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 enum pe__bundle_mount_flags {
  24     pe__bundle_mount_none       = 0x00,
  25 
  26     // mount instance-specific subdirectory rather than source directly
  27     pe__bundle_mount_subdir     = 0x01
  28 };
  29 
  30 typedef struct {
  31     char *source;
  32     char *target;
  33     char *options;
  34     uint32_t flags; // bitmask of pe__bundle_mount_flags
  35 } pe__bundle_mount_t;
  36 
  37 typedef struct {
  38     char *source;
  39     char *target;
  40 } pe__bundle_port_t;
  41 
  42 enum pe__container_agent {
  43     PE__CONTAINER_AGENT_UNKNOWN,
  44     PE__CONTAINER_AGENT_DOCKER,
  45     PE__CONTAINER_AGENT_RKT,
  46     PE__CONTAINER_AGENT_PODMAN,
  47 };
  48 
  49 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
  50 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
  51 #define PE__CONTAINER_AGENT_RKT_S     "rkt"
  52 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
  53 
  54 typedef struct pe__bundle_variant_data_s {
  55         int promoted_max;
  56         int nreplicas;
  57         int nreplicas_per_host;
  58         char *prefix;
  59         char *image;
  60         const char *ip_last;
  61         char *host_network;
  62         char *host_netmask;
  63         char *control_port;
  64         char *container_network;
  65         char *ip_range_start;
  66         gboolean add_host;
  67         gchar *container_host_options;
  68         char *container_command;
  69         char *launcher_options;
  70         const char *attribute_target;
  71 
  72         pcmk_resource_t *child;
  73 
  74         GList *replicas;    // pcmk__bundle_replica_t *
  75         GList *ports;       // pe__bundle_port_t *
  76         GList *mounts;      // pe__bundle_mount_t *
  77 
  78         enum pe__container_agent agent_type;
  79 } pe__bundle_variant_data_t;
  80 
  81 #define get_bundle_variant_data(data, rsc)                                  \
  82     pcmk__assert(pcmk__is_bundle(rsc) && (rsc->variant_opaque != NULL));    \
  83     data = (pe__bundle_variant_data_t *) rsc->variant_opaque;
  84 
  85 /*!
  86  * \internal
  87  * \brief Get maximum number of bundle replicas allowed to run
  88  *
  89  * \param[in] rsc  Bundle or bundled resource to check
  90  *
  91  * \return Maximum replicas for bundle corresponding to \p rsc
  92  */
  93 int
  94 pe__bundle_max(const pcmk_resource_t *rsc)
     /*  */
  95 {
  96     const pe__bundle_variant_data_t *bundle_data = NULL;
  97 
  98     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
  99     return bundle_data->nreplicas;
 100 }
 101 
 102 /*!
 103  * \internal
 104  * \brief Get the resource inside a bundle
 105  *
 106  * \param[in] bundle  Bundle to check
 107  *
 108  * \return Resource inside \p bundle if any, otherwise NULL
 109  */
 110 pcmk_resource_t *
 111 pe__bundled_resource(const pcmk_resource_t *rsc)
     /*  */
 112 {
 113     const pe__bundle_variant_data_t *bundle_data = NULL;
 114 
 115     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 116     return bundle_data->child;
 117 }
 118 
 119 /*!
 120  * \internal
 121  * \brief Get containerized resource corresponding to a given bundle container
 122  *
 123  * \param[in] instance  Collective instance that might be a bundle container
 124  *
 125  * \return Bundled resource instance inside \p instance if it is a bundle
 126  *         container instance, otherwise NULL
 127  */
 128 const pcmk_resource_t *
 129 pe__get_rsc_in_container(const pcmk_resource_t *instance)
     /*  */
 130 {
 131     const pe__bundle_variant_data_t *data = NULL;
 132     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 133 
 134     if (!pcmk__is_bundle(top)) {
 135         return NULL;
 136     }
 137     get_bundle_variant_data(data, top);
 138 
 139     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 140         const pcmk__bundle_replica_t *replica = iter->data;
 141 
 142         if (instance == replica->container) {
 143             return replica->child;
 144         }
 145     }
 146     return NULL;
 147 }
 148 
 149 /*!
 150  * \internal
 151  * \brief Check whether a given node is created by a bundle
 152  *
 153  * \param[in] bundle  Bundle resource to check
 154  * \param[in] node    Node to check
 155  *
 156  * \return true if \p node is an instance of \p bundle, otherwise false
 157  */
 158 bool
 159 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
     /*  */
 160                             const pcmk_node_t *node)
 161 {
 162     pe__bundle_variant_data_t *bundle_data = NULL;
 163 
 164     get_bundle_variant_data(bundle_data, bundle);
 165     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 166         pcmk__bundle_replica_t *replica = iter->data;
 167 
 168         if (pcmk__same_node(node, replica->node)) {
 169             return true;
 170         }
 171     }
 172     return false;
 173 }
 174 
 175 /*!
 176  * \internal
 177  * \brief Get the container of a bundle's first replica
 178  *
 179  * \param[in] bundle  Bundle resource to get container for
 180  *
 181  * \return Container resource from first replica of \p bundle if any,
 182  *         otherwise NULL
 183  */
 184 pcmk_resource_t *
 185 pe__first_container(const pcmk_resource_t *bundle)
     /*  */
 186 {
 187     const pe__bundle_variant_data_t *bundle_data = NULL;
 188     const pcmk__bundle_replica_t *replica = NULL;
 189 
 190     get_bundle_variant_data(bundle_data, bundle);
 191     if (bundle_data->replicas == NULL) {
 192         return NULL;
 193     }
 194     replica = bundle_data->replicas->data;
 195     return replica->container;
 196 }
 197 
 198 /*!
 199  * \internal
 200  * \brief Iterate over bundle replicas
 201  *
 202  * \param[in,out] bundle     Bundle to iterate over
 203  * \param[in]     fn         Function to call for each replica (its return value
 204  *                           indicates whether to continue iterating)
 205  * \param[in,out] user_data  Pointer to pass to \p fn
 206  */
 207 void
 208 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
     /*  */
 209                            bool (*fn)(pcmk__bundle_replica_t *, void *),
 210                            void *user_data)
 211 {
 212     const pe__bundle_variant_data_t *bundle_data = NULL;
 213 
 214     get_bundle_variant_data(bundle_data, bundle);
 215     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 216         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
 217             break;
 218         }
 219     }
 220 }
 221 
 222 /*!
 223  * \internal
 224  * \brief Iterate over const bundle replicas
 225  *
 226  * \param[in]     bundle     Bundle to iterate over
 227  * \param[in]     fn         Function to call for each replica (its return value
 228  *                           indicates whether to continue iterating)
 229  * \param[in,out] user_data  Pointer to pass to \p fn
 230  */
 231 void
 232 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
     /*  */
 233                                  bool (*fn)(const pcmk__bundle_replica_t *,
 234                                             void *),
 235                                  void *user_data)
 236 {
 237     const pe__bundle_variant_data_t *bundle_data = NULL;
 238 
 239     get_bundle_variant_data(bundle_data, bundle);
 240     for (const GList *iter = bundle_data->replicas; iter != NULL;
 241          iter = iter->next) {
 242 
 243         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
 244             break;
 245         }
 246     }
 247 }
 248 
 249 static char *
 250 next_ip(const char *last_ip)
     /*  */
 251 {
 252     unsigned int oct1 = 0;
 253     unsigned int oct2 = 0;
 254     unsigned int oct3 = 0;
 255     unsigned int oct4 = 0;
 256     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 257 
 258     if (rc != 4) {
 259         /*@ TODO check for IPv6 */
 260         return NULL;
 261 
 262     } else if (oct3 > 253) {
 263         return NULL;
 264 
 265     } else if (oct4 > 253) {
 266         ++oct3;
 267         oct4 = 1;
 268 
 269     } else {
 270         ++oct4;
 271     }
 272 
 273     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 274 }
 275 
 276 static void
 277 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
     /*  */
 278             GString *buffer)
 279 {
 280     if(data->ip_range_start == NULL) {
 281         return;
 282 
 283     } else if(data->ip_last) {
 284         replica->ipaddr = next_ip(data->ip_last);
 285 
 286     } else {
 287         replica->ipaddr = strdup(data->ip_range_start);
 288     }
 289 
 290     data->ip_last = replica->ipaddr;
 291     switch (data->agent_type) {
 292         case PE__CONTAINER_AGENT_DOCKER:
 293         case PE__CONTAINER_AGENT_PODMAN:
 294             if (data->add_host) {
 295                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 296                                        data->prefix, replica->offset,
 297                                        replica->ipaddr);
 298             } else {
 299                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 300                                        replica->ipaddr, data->prefix,
 301                                        replica->offset);
 302             }
 303             break;
 304 
 305         case PE__CONTAINER_AGENT_RKT:
 306             g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 307                                    replica->ipaddr, data->prefix,
 308                                    replica->offset);
 309             break;
 310 
 311         default: // PE__CONTAINER_AGENT_UNKNOWN
 312             break;
 313     }
 314 }
 315 
 316 static xmlNode *
 317 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 318 {
 319     xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 320 
 321     crm_xml_add(rsc, PCMK_XA_ID, name);
 322     crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
 323     crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
 324     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 325 
 326     return rsc;
 327 }
 328 
 329 /*!
 330  * \internal
 331  * \brief Check whether cluster can manage resource inside container
 332  *
 333  * \param[in,out] data  Container variant data
 334  *
 335  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 336  *
 337  * \note The resource is manageable if an IP range or control port has been
 338  *       specified. If a control port is used without an IP range, replicas per
 339  *       host must be 1.
 340  */
 341 static bool
 342 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 343 {
 344     if(data->ip_range_start) {
 345         return TRUE;
 346     }
 347     if(data->control_port) {
 348         if(data->nreplicas_per_host > 1) {
 349             pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
 350                              "requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
 351                              data->prefix);
 352             data->nreplicas_per_host = 1;
 353             // @TODO to be sure:
 354             // pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
 355         }
 356         return TRUE;
 357     }
 358     return FALSE;
 359 }
 360 
 361 static int
 362 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 363                    pcmk__bundle_replica_t *replica)
 364 {
 365     if(data->ip_range_start) {
 366         char *id = NULL;
 367         xmlNode *xml_ip = NULL;
 368         xmlNode *xml_obj = NULL;
 369 
 370         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 371         crm_xml_sanitize_id(id);
 372         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 373         free(id);
 374 
 375         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
 376         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 377                        data->prefix, replica->offset);
 378 
 379         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 380         if(data->host_network) {
 381             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 382         }
 383 
 384         if(data->host_netmask) {
 385             crm_create_nvpair_xml(xml_obj, NULL,
 386                                   "cidr_netmask", data->host_netmask);
 387 
 388         } else {
 389             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 390         }
 391 
 392         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
 393         crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
 394                           "60s", NULL);
 395 
 396         // TODO: Other ops? Timeouts and intervals from underlying resource?
 397 
 398         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 399                                 parent->cluster) != pcmk_rc_ok) {
 400             return pcmk_rc_unpack_error;
 401         }
 402 
 403         parent->children = g_list_append(parent->children, replica->ip);
 404     }
 405     return pcmk_rc_ok;
 406 }
 407 
 408 static const char*
 409 container_agent_str(enum pe__container_agent t)
     /*  */
 410 {
 411     switch (t) {
 412         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 413         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
 414         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 415         default: // PE__CONTAINER_AGENT_UNKNOWN
 416             break;
 417     }
 418     return PE__CONTAINER_AGENT_UNKNOWN_S;
 419 }
 420 
 421 static int
 422 create_container_resource(pcmk_resource_t *parent,
     /*  */
 423                           const pe__bundle_variant_data_t *data,
 424                           pcmk__bundle_replica_t *replica)
 425 {
 426     char *id = NULL;
 427     xmlNode *xml_container = NULL;
 428     xmlNode *xml_obj = NULL;
 429 
 430     // Agent-specific
 431     const char *hostname_opt = NULL;
 432     const char *env_opt = NULL;
 433     const char *agent_str = NULL;
 434     int volid = 0;  // rkt-only
 435 
 436     GString *buffer = NULL;
 437     GString *dbuffer = NULL;
 438 
 439     // Where syntax differences are drop-in replacements, set them now
 440     switch (data->agent_type) {
 441         case PE__CONTAINER_AGENT_DOCKER:
 442         case PE__CONTAINER_AGENT_PODMAN:
 443             hostname_opt = "-h ";
 444             env_opt = "-e ";
 445             break;
 446         case PE__CONTAINER_AGENT_RKT:
 447             hostname_opt = "--hostname=";
 448             env_opt = "--environment=";
 449             break;
 450         default:    // PE__CONTAINER_AGENT_UNKNOWN
 451             return pcmk_rc_unpack_error;
 452     }
 453     agent_str = container_agent_str(data->agent_type);
 454 
 455     buffer = g_string_sized_new(4096);
 456 
 457     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 458                            replica->offset);
 459     crm_xml_sanitize_id(id);
 460     xml_container = create_resource(id, "heartbeat", agent_str);
 461     free(id);
 462 
 463     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
 464     crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 465 
 466     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 467     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
 468     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
 469     crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);
 470 
 471     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 472         g_string_append(buffer, " --restart=no");
 473     }
 474 
 475     /* Set a container hostname only if we have an IP to map it to. The user can
 476      * set -h or --uts=host themselves if they want a nicer name for logs, but
 477      * this makes applications happy who need their  hostname to match the IP
 478      * they bind to.
 479      */
 480     if (data->ip_range_start != NULL) {
 481         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 482                                replica->offset);
 483     }
 484     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 485 
 486     if (data->container_network != NULL) {
 487         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 488     }
 489 
 490     if (data->control_port != NULL) {
 491         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
 492                        data->control_port, NULL);
 493     } else {
 494         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
 495                                env_opt, DEFAULT_REMOTE_PORT);
 496     }
 497 
 498     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 499         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 500         char *source = NULL;
 501 
 502         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 503             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 504                                        replica->offset);
 505             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 506         }
 507 
 508         switch (data->agent_type) {
 509             case PE__CONTAINER_AGENT_DOCKER:
 510             case PE__CONTAINER_AGENT_PODMAN:
 511                 pcmk__g_strcat(buffer,
 512                                " -v ", pcmk__s(source, mount->source),
 513                                ":", mount->target, NULL);
 514 
 515                 if (mount->options != NULL) {
 516                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 517                 }
 518                 break;
 519             case PE__CONTAINER_AGENT_RKT:
 520                 g_string_append_printf(buffer,
 521                                        " --volume vol%d,kind=host,"
 522                                        "source=%s%s%s "
 523                                        "--mount volume=vol%d,target=%s",
 524                                        volid, pcmk__s(source, mount->source),
 525                                        (mount->options != NULL)? "," : "",
 526                                        pcmk__s(mount->options, ""),
 527                                        volid, mount->target);
 528                 volid++;
 529                 break;
 530             default:
 531                 break;
 532         }
 533         free(source);
 534     }
 535 
 536     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 537         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 538 
 539         switch (data->agent_type) {
 540             case PE__CONTAINER_AGENT_DOCKER:
 541             case PE__CONTAINER_AGENT_PODMAN:
 542                 if (replica->ipaddr != NULL) {
 543                     pcmk__g_strcat(buffer,
 544                                    " -p ", replica->ipaddr, ":", port->source,
 545                                    ":", port->target, NULL);
 546 
 547                 } else if (!pcmk__str_eq(data->container_network,
 548                                          PCMK_VALUE_HOST, pcmk__str_none)) {
 549                     // No need to do port mapping if net == host
 550                     pcmk__g_strcat(buffer,
 551                                    " -p ", port->source, ":", port->target,
 552                                    NULL);
 553                 }
 554                 break;
 555             case PE__CONTAINER_AGENT_RKT:
 556                 if (replica->ipaddr != NULL) {
 557                     pcmk__g_strcat(buffer,
 558                                    " --port=", port->target,
 559                                    ":", replica->ipaddr, ":", port->source,
 560                                    NULL);
 561                 } else {
 562                     pcmk__g_strcat(buffer,
 563                                    " --port=", port->target, ":", port->source,
 564                                    NULL);
 565                 }
 566                 break;
 567             default:
 568                 break;
 569         }
 570     }
 571 
 572     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 573      * it would cause restarts during rolling upgrades.
 574      *
 575      * In a previous version of the container resource creation logic, if
 576      * data->launcher_options is not NULL, we append
 577      * (" %s", data->launcher_options) even if data->launcher_options is an
 578      * empty string. Likewise for data->container_host_options. Using
 579      *
 580      *     pcmk__add_word(buffer, 0, data->launcher_options)
 581      *
 582      * removes that extra trailing space, causing a resource definition change.
 583      */
 584     if (data->launcher_options != NULL) {
 585         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 586     }
 587 
 588     if (data->container_host_options != NULL) {
 589         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 590     }
 591 
 592     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 593                           (const char *) buffer->str);
 594     g_string_free(buffer, TRUE);
 595 
 596     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 597                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 598     if (dbuffer != NULL) {
 599         g_string_free(dbuffer, TRUE);
 600     }
 601 
 602     if (replica->child != NULL) {
 603         if (data->container_command != NULL) {
 604             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 605                                   data->container_command);
 606         } else {
 607             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 608                                   SBIN_DIR "/pacemaker-remoted");
 609         }
 610 
 611         /* TODO: Allow users to specify their own?
 612          *
 613          * We just want to know if the container is alive; we'll monitor the
 614          * child independently.
 615          */
 616         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 617 #if 0
 618         /* @TODO Consider supporting the use case where we can start and stop
 619          * resources, but not proxy local commands (such as setting node
 620          * attributes), by running the local executor in stand-alone mode.
 621          * However, this would probably be better done via ACLs as with other
 622          * Pacemaker Remote nodes.
 623          */
 624     } else if ((child != NULL) && data->untrusted) {
 625         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 626                               CRM_DAEMON_DIR "/pacemaker-execd");
 627         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 628                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 629 #endif
 630     } else {
 631         if (data->container_command != NULL) {
 632             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 633                                   data->container_command);
 634         }
 635 
 636         /* TODO: Allow users to specify their own?
 637          *
 638          * We don't know what's in the container, so we just want to know if it
 639          * is alive.
 640          */
 641         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 642     }
 643 
 644     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
 645     crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
 646                       "60s", NULL);
 647 
 648     // TODO: Other ops? Timeouts and intervals from underlying resource?
 649     if (pe__unpack_resource(xml_container, &replica->container, parent,
 650                             parent->cluster) != pcmk_rc_ok) {
 651         return pcmk_rc_unpack_error;
 652     }
 653     pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
 654     parent->children = g_list_append(parent->children, replica->container);
 655 
 656     return pcmk_rc_ok;
 657 }
 658 
 659 /*!
 660  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 661  *
 662  * \param[in,out] rsc    Resource to modify
 663  * \param[in]     uname  Name of node to ban
 664  */
 665 static void
 666 disallow_node(pcmk_resource_t *rsc, const char *uname)
     /*  */
 667 {
 668     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 669 
 670     if (match) {
 671         ((pcmk_node_t *) match)->weight = -PCMK_SCORE_INFINITY;
 672         ((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
 673     }
 674     if (rsc->children) {
 675         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 676     }
 677 }
 678 
 679 static int
 680 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 681                        pcmk__bundle_replica_t *replica)
 682 {
 683     if (replica->child && valid_network(data)) {
 684         GHashTableIter gIter;
 685         pcmk_node_t *node = NULL;
 686         xmlNode *xml_remote = NULL;
 687         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 688         char *port_s = NULL;
 689         const char *uname = NULL;
 690         const char *connect_name = NULL;
 691 
 692         if (pe_find_resource(parent->cluster->resources, id) != NULL) {
 693             free(id);
 694             // The biggest hammer we have
 695             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 696                                    replica->child->id, replica->offset);
 697             //@TODO return error instead of asserting?
 698             pcmk__assert(pe_find_resource(parent->cluster->resources,
 699                                           id) == NULL);
 700         }
 701 
 702         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 703          * connection does not have its own IP is a magic string that we use to
 704          * support nested remotes (i.e. a bundle running on a remote node).
 705          */
 706         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 707 
 708         if (data->control_port == NULL) {
 709             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 710         }
 711 
 712         /* This sets replica->container as replica->remote's container, which is
 713          * similar to what happens with guest nodes. This is how the scheduler
 714          * knows that the bundle node is fenced by recovering the container, and
 715          * that remote should be ordered relative to the container.
 716          */
 717         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 718                                           NULL, NULL, NULL,
 719                                           connect_name, (data->control_port?
 720                                           data->control_port : port_s));
 721         free(port_s);
 722 
 723         /* Abandon our created ID, and pull the copy from the XML, because we
 724          * need something that will get freed during scheduler data cleanup to
 725          * use as the node ID and uname.
 726          */
 727         free(id);
 728         id = NULL;
 729         uname = pcmk__xe_id(xml_remote);
 730 
 731         /* Ensure a node has been created for the guest (it may have already
 732          * been, if it has a permanent node attribute), and ensure its weight is
 733          * -INFINITY so no other resources can run on it.
 734          */
 735         node = pcmk_find_node(parent->cluster, uname);
 736         if (node == NULL) {
 737             node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
 738                                   -PCMK_SCORE_INFINITY, parent->cluster);
 739         } else {
 740             node->weight = -PCMK_SCORE_INFINITY;
 741         }
 742         node->rsc_discover_mode = pcmk_probe_never;
 743 
 744         /* unpack_remote_nodes() ensures that each remote node and guest node
 745          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
 746          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
 747          * obvious what nodes will be needed, so we do it just above.
 748          *
 749          * Worse, that means that the node may have been utilized while
 750          * unpacking other resources, without our weight correction. The most
 751          * likely place for this to happen is when pe__unpack_resource() calls
 752          * resource_location() to set a default score in symmetric clusters.
 753          * This adds a node *copy* to each resource's allowed nodes, and these
 754          * copies will have the wrong weight.
 755          *
 756          * As a hacky workaround, fix those copies here.
 757          *
 758          * @TODO Possible alternative: ensure bundles are unpacked before other
 759          * resources, so the weight is correct before any copies are made.
 760          */
 761         g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
 762                        (gpointer) uname);
 763 
 764         replica->node = pe__copy_node(node);
 765         replica->node->weight = 500;
 766         replica->node->rsc_discover_mode = pcmk_probe_exclusive;
 767 
 768         /* Ensure the node shows up as allowed and with the correct discovery set */
 769         if (replica->child->allowed_nodes != NULL) {
 770             g_hash_table_destroy(replica->child->allowed_nodes);
 771         }
 772         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 773         g_hash_table_insert(replica->child->allowed_nodes,
 774                             (gpointer) replica->node->details->id,
 775                             pe__copy_node(replica->node));
 776 
 777         {
 778             pcmk_node_t *copy = pe__copy_node(replica->node);
 779             copy->weight = -PCMK_SCORE_INFINITY;
 780             g_hash_table_insert(replica->child->parent->allowed_nodes,
 781                                 (gpointer) replica->node->details->id, copy);
 782         }
 783         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 784                                 parent->cluster) != pcmk_rc_ok) {
 785             return pcmk_rc_unpack_error;
 786         }
 787 
 788         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 789         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 790             if (pcmk__is_pacemaker_remote_node(node)) {
 791                 /* Remote resources can only run on 'normal' cluster node */
 792                 node->weight = -PCMK_SCORE_INFINITY;
 793             }
 794         }
 795 
 796         replica->node->details->remote_rsc = replica->remote;
 797 
 798         // Ensure pcmk__is_guest_or_bundle_node() functions correctly
 799         replica->remote->container = replica->container;
 800 
 801         /* A bundle's #kind is closer to "container" (guest node) than the
 802          * "remote" set by pe_create_node().
 803          */
 804         pcmk__insert_dup(replica->node->details->attrs,
 805                          CRM_ATTR_KIND, "container");
 806 
 807         /* One effect of this is that setup_container() will add
 808          * replica->remote to replica->container's fillers, which will make
 809          * pe__resource_contains_guest_node() true for replica->container.
 810          *
 811          * replica->child does NOT get added to replica->container's fillers.
 812          * The only noticeable effect if it did would be for its fail count to
 813          * be taken into account when checking replica->container's migration
 814          * threshold.
 815          */
 816         parent->children = g_list_append(parent->children, replica->remote);
 817     }
 818     return pcmk_rc_ok;
 819 }
 820 
 821 static int
 822 create_replica_resources(pcmk_resource_t *parent,
     /*  */
 823                          pe__bundle_variant_data_t *data,
 824                          pcmk__bundle_replica_t *replica)
 825 {
 826     int rc = pcmk_rc_ok;
 827 
 828     rc = create_container_resource(parent, data, replica);
 829     if (rc != pcmk_rc_ok) {
 830         return rc;
 831     }
 832 
 833     rc = create_ip_resource(parent, data, replica);
 834     if (rc != pcmk_rc_ok) {
 835         return rc;
 836     }
 837 
 838     rc = create_remote_resource(parent, data, replica);
 839     if (rc != pcmk_rc_ok) {
 840         return rc;
 841     }
 842 
 843     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 844         pcmk__insert_meta(replica->child, "external-ip", replica->ipaddr);
 845     }
 846 
 847     if (replica->remote != NULL) {
 848         /*
 849          * Allow the remote connection resource to be allocated to a
 850          * different node than the one on which the container is active.
 851          *
 852          * This makes it possible to have Pacemaker Remote nodes running
 853          * containers with pacemaker-remoted inside in order to start
 854          * services inside those containers.
 855          */
 856         pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
 857     }
 858     return rc;
 859 }
 860 
 861 static void
 862 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 863           const char *target, const char *options, uint32_t flags)
 864 {
 865     pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
 866                                                    sizeof(pe__bundle_mount_t));
 867 
 868     mount->source = pcmk__str_copy(source);
 869     mount->target = pcmk__str_copy(target);
 870     mount->options = pcmk__str_copy(options);
 871     mount->flags = flags;
 872     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 873 }
 874 
 875 static void
 876 mount_free(pe__bundle_mount_t *mount)
     /*  */
 877 {
 878     free(mount->source);
 879     free(mount->target);
 880     free(mount->options);
 881     free(mount);
 882 }
 883 
 884 static void
 885 port_free(pe__bundle_port_t *port)
     /*  */
 886 {
 887     free(port->source);
 888     free(port->target);
 889     free(port);
 890 }
 891 
 892 static pcmk__bundle_replica_t *
 893 replica_for_remote(pcmk_resource_t *remote)
     /*  */
 894 {
 895     pcmk_resource_t *top = remote;
 896     pe__bundle_variant_data_t *bundle_data = NULL;
 897 
 898     if (top == NULL) {
 899         return NULL;
 900     }
 901 
 902     while (top->parent != NULL) {
 903         top = top->parent;
 904     }
 905 
 906     get_bundle_variant_data(bundle_data, top);
 907     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 908          gIter = gIter->next) {
 909         pcmk__bundle_replica_t *replica = gIter->data;
 910 
 911         if (replica->remote == remote) {
 912             return replica;
 913         }
 914     }
 915     CRM_LOG_ASSERT(FALSE);
 916     return NULL;
 917 }
 918 
 919 bool
 920 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
     /*  */
 921 {
 922     const char *value;
 923     GHashTable *params = NULL;
 924 
 925     if (rsc == NULL) {
 926         return false;
 927     }
 928 
 929     // Use NULL node since pcmk__bundle_expand() uses that to set value
 930     params = pe_rsc_params(rsc, NULL, rsc->cluster);
 931     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 932 
 933     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 934            && xml_contains_remote_node(rsc->xml);
 935 }
 936 
 937 const char *
 938 pe__add_bundle_remote_name(pcmk_resource_t *rsc, xmlNode *xml,
     /*  */
 939                            const char *field)
 940 {
 941     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 942 
 943     pcmk_node_t *node = NULL;
 944     pcmk__bundle_replica_t *replica = NULL;
 945 
 946     if (!pe__bundle_needs_remote_name(rsc)) {
 947         return NULL;
 948     }
 949 
 950     replica = replica_for_remote(rsc);
 951     if (replica == NULL) {
 952         return NULL;
 953     }
 954 
 955     node = replica->container->allocated_to;
 956     if (node == NULL) {
 957         /* If it won't be running anywhere after the
 958          * transition, go with where it's running now.
 959          */
 960         node = pcmk__current_node(replica->container);
 961     }
 962 
 963     if(node == NULL) {
 964         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 965         return NULL;
 966     }
 967 
 968     crm_trace("Setting address for bundle connection %s to bundle host %s",
 969               rsc->id, pcmk__node_name(node));
 970     if(xml != NULL && field != NULL) {
 971         crm_xml_add(xml, field, node->details->uname);
 972     }
 973 
 974     return node->details->uname;
 975 }
 976 
 977 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 978         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 979                                    "Bundle mount", pcmk__xe_id(mount_xml),  \
 980                                    flags, (flags_to_set), #flags_to_set);   \
 981     } while (0)
 982 
 983 gboolean
 984 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
     /*  */
 985 {
 986     const char *value = NULL;
 987     xmlNode *xml_obj = NULL;
 988     const xmlNode *xml_child = NULL;
 989     xmlNode *xml_resource = NULL;
 990     pe__bundle_variant_data_t *bundle_data = NULL;
 991     bool need_log_mount = TRUE;
 992 
 993     pcmk__assert(rsc != NULL);
 994     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 995 
 996     bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
 997     rsc->variant_opaque = bundle_data;
 998     bundle_data->prefix = strdup(rsc->id);
 999 
1000     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_DOCKER, NULL, NULL);
1001     if (xml_obj != NULL) {
1002         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1003     } else {
1004         xml_obj = pcmk__xe_first_child(rsc->xml, PCMK__XE_RKT, NULL, NULL);
1005         if (xml_obj != NULL) {
1006             pcmk__warn_once(pcmk__wo_rkt,
1007                             "Support for " PCMK__XE_RKT " in bundles "
1008                             "(such as %s) is deprecated and will be "
1009                             "removed in a future release", rsc->id);
1010             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1011         } else {
1012             xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PODMAN, NULL,
1013                                            NULL);
1014             if (xml_obj != NULL) {
1015                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1016             } else {
1017                 return FALSE;
1018             }
1019         }
1020     }
1021 
1022     // Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
1023     value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
1024     if (value == NULL) {
1025         // @COMPAT deprecated since 2.0.0
1026         value = crm_element_value(xml_obj, PCMK__XA_PROMOTED_MAX_LEGACY);
1027 
1028         if (value != NULL) {
1029             pcmk__warn_once(pcmk__wo_bundle_master,
1030                             "Support for the " PCMK__XA_PROMOTED_MAX_LEGACY
1031                             " attribute (such as in %s) is deprecated and "
1032                             "will be removed in a future release. Use "
1033                             PCMK_XA_PROMOTED_MAX " instead.",
1034                             rsc->id);
1035         }
1036     }
1037     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1038 
1039     /* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
1040      * otherwise
1041      */
1042     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
1043     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1044         bundle_data->nreplicas = bundle_data->promoted_max;
1045     } else {
1046         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1047     }
1048 
1049     /*
1050      * Communication between containers on the same host via the
1051      * floating IPs only works if the container is started with:
1052      *   --userland-proxy=false --ip-masq=false
1053      */
1054     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
1055     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1056     if (bundle_data->nreplicas_per_host == 1) {
1057         pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
1058     }
1059 
1060     bundle_data->container_command =
1061         crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
1062     bundle_data->launcher_options = crm_element_value_copy(xml_obj,
1063                                                            PCMK_XA_OPTIONS);
1064     bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
1065     bundle_data->container_network = crm_element_value_copy(xml_obj,
1066                                                             PCMK_XA_NETWORK);
1067 
1068     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_NETWORK, NULL, NULL);
1069     if(xml_obj) {
1070         bundle_data->ip_range_start =
1071             crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
1072         bundle_data->host_netmask =
1073             crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
1074         bundle_data->host_network =
1075             crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
1076         bundle_data->control_port =
1077             crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
1078         value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
1079         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1080             bundle_data->add_host = TRUE;
1081         }
1082 
1083         for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
1084                                               NULL, NULL);
1085              xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {
1086 
1087             pe__bundle_port_t *port =
1088                 pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1089 
1090             port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);
1091 
1092             if(port->source == NULL) {
1093                 port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
1094             } else {
1095                 port->target = crm_element_value_copy(xml_child,
1096                                                       PCMK_XA_INTERNAL_PORT);
1097             }
1098 
1099             if(port->source != NULL && strlen(port->source) > 0) {
1100                 if(port->target == NULL) {
1101                     port->target = strdup(port->source);
1102                 }
1103                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1104 
1105             } else {
1106                 pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
1107                                  pcmk__xe_id(xml_child));
1108                 port_free(port);
1109             }
1110         }
1111     }
1112 
1113     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_STORAGE, NULL, NULL);
1114     for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
1115                                           NULL, NULL);
1116          xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {
1117 
1118         const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
1119         const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
1120         const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
1121         int flags = pe__bundle_mount_none;
1122 
1123         if (source == NULL) {
1124             source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
1125             pe__set_bundle_mount_flags(xml_child, flags,
1126                                        pe__bundle_mount_subdir);
1127         }
1128 
1129         if (source && target) {
1130             mount_add(bundle_data, source, target, options, flags);
1131             if (strcmp(target, "/var/log") == 0) {
1132                 need_log_mount = FALSE;
1133             }
1134         } else {
1135             pcmk__config_err("Invalid mount directive %s",
1136                              pcmk__xe_id(xml_child));
1137         }
1138     }
1139 
1140     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PRIMITIVE, NULL, NULL);
1141     if (xml_obj && valid_network(bundle_data)) {
1142         char *value = NULL;
1143         xmlNode *xml_set = NULL;
1144 
1145         xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);
1146 
1147         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1148          * part of the resource name, so that bundles don't restart in a rolling
1149          * upgrade. (It also avoids needing to change regression tests.)
1150          */
1151         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1152                       (bundle_data->promoted_max? "master"
1153                       : (const char *)xml_resource->name));
1154 
1155         xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
1156         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1157 
1158         crm_create_nvpair_xml(xml_set, NULL,
1159                               PCMK_META_ORDERED, PCMK_VALUE_TRUE);
1160 
1161         value = pcmk__itoa(bundle_data->nreplicas);
1162         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
1163         free(value);
1164 
1165         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1166         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
1167         free(value);
1168 
1169         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
1170                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1171 
1172         if (bundle_data->promoted_max) {
1173             crm_create_nvpair_xml(xml_set, NULL,
1174                                   PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);
1175 
1176             value = pcmk__itoa(bundle_data->promoted_max);
1177             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
1178             free(value);
1179         }
1180 
1181         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
1182         pcmk__xml_copy(xml_resource, xml_obj);
1183 
1184     } else if(xml_obj) {
1185         pcmk__config_err("Cannot control %s inside %s without either "
1186                          PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
1187                          rsc->id, pcmk__xe_id(xml_obj));
1188         return FALSE;
1189     }
1190 
1191     if(xml_resource) {
1192         int lpc = 0;
1193         GList *childIter = NULL;
1194         pe__bundle_port_t *port = NULL;
1195         GString *buffer = NULL;
1196 
1197         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
1198                                 scheduler) != pcmk_rc_ok) {
1199             return FALSE;
1200         }
1201 
1202         /* Currently, we always map the default authentication key location
1203          * into the same location inside the container.
1204          *
1205          * Ideally, we would respect the host's PCMK_authkey_location, but:
1206          * - it may be different on different nodes;
1207          * - the actual connection will do extra checking to make sure the key
1208          *   file exists and is readable, that we can't do here on the DC
1209          * - tools such as crm_resource and crm_simulate may not have the same
1210          *   environment variables as the cluster, causing operation digests to
1211          *   differ
1212          *
1213          * Always using the default location inside the container is fine,
1214          * because we control the pacemaker_remote environment, and it avoids
1215          * having to pass another environment variable to the container.
1216          *
1217          * @TODO A better solution may be to have only pacemaker_remote use the
1218          * environment variable, and have the cluster nodes use a new
1219          * cluster option for key location. This would introduce the limitation
1220          * of the location being the same on all cluster nodes, but that's
1221          * reasonable.
1222          */
1223         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1224                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1225 
1226         if (need_log_mount) {
1227             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1228                       pe__bundle_mount_subdir);
1229         }
1230 
1231         port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1232         if(bundle_data->control_port) {
1233             port->source = strdup(bundle_data->control_port);
1234         } else {
1235             /* If we wanted to respect PCMK_remote_port, we could use
1236              * crm_default_remote_port() here and elsewhere in this file instead
1237              * of DEFAULT_REMOTE_PORT.
1238              *
1239              * However, it gains nothing, since we control both the container
1240              * environment and the connection resource parameters, and the user
1241              * can use a different port if desired by setting
1242              * PCMK_XA_CONTROL_PORT.
1243              */
1244             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1245         }
1246         port->target = strdup(port->source);
1247         bundle_data->ports = g_list_append(bundle_data->ports, port);
1248 
1249         buffer = g_string_sized_new(1024);
1250         for (childIter = bundle_data->child->children; childIter != NULL;
1251              childIter = childIter->next) {
1252 
1253             pcmk__bundle_replica_t *replica = NULL;
1254 
1255             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1256             replica->child = childIter->data;
1257             replica->child->exclusive_discover = TRUE;
1258             replica->offset = lpc++;
1259 
1260             // Ensure the child's notify gets set based on the underlying primitive's value
1261             if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
1262                 pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
1263             }
1264 
1265             allocate_ip(bundle_data, replica, buffer);
1266             bundle_data->replicas = g_list_append(bundle_data->replicas,
1267                                                   replica);
1268             bundle_data->attribute_target =
1269                 g_hash_table_lookup(replica->child->meta,
1270                                     PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
1271         }
1272         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1273 
1274         if (bundle_data->attribute_target) {
1275             pcmk__insert_dup(rsc->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1276                              bundle_data->attribute_target);
1277             pcmk__insert_dup(bundle_data->child->meta,
1278                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1279                              bundle_data->attribute_target);
1280         }
1281 
1282     } else {
1283         // Just a naked container, no pacemaker-remote
1284         GString *buffer = g_string_sized_new(1024);
1285 
1286         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1287             pcmk__bundle_replica_t *replica = NULL;
1288 
1289             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1290             replica->offset = lpc;
1291             allocate_ip(bundle_data, replica, buffer);
1292             bundle_data->replicas = g_list_append(bundle_data->replicas,
1293                                                   replica);
1294         }
1295         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1296     }
1297 
1298     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1299          gIter = gIter->next) {
1300         pcmk__bundle_replica_t *replica = gIter->data;
1301 
1302         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1303             pcmk__config_err("Failed unpacking resource %s", rsc->id);
1304             rsc->fns->free(rsc);
1305             return FALSE;
1306         }
1307 
1308         /* Utilization needs special handling for bundles. It makes no sense for
1309          * the inner primitive to have utilization, because it is tied
1310          * one-to-one to the guest node created by the container resource -- and
1311          * there's no way to set capacities for that guest node anyway.
1312          *
1313          * What the user really wants is to configure utilization for the
1314          * container. However, the schema only allows utilization for
1315          * primitives, and the container resource is implicit anyway, so the
1316          * user can *only* configure utilization for the inner primitive. If
1317          * they do, move the primitive's utilization values to the container.
1318          *
1319          * @TODO This means that bundles without an inner primitive can't have
1320          * utilization. An alternative might be to allow utilization values in
1321          * the top-level bundle XML in the schema, and copy those to each
1322          * container.
1323          */
1324         if (replica->child != NULL) {
1325             GHashTable *empty = replica->container->utilization;
1326 
1327             replica->container->utilization = replica->child->utilization;
1328             replica->child->utilization = empty;
1329         }
1330     }
1331 
1332     if (bundle_data->child) {
1333         rsc->children = g_list_append(rsc->children, bundle_data->child);
1334     }
1335     return TRUE;
1336 }
1337 
1338 static int
1339 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1340 {
1341     if (rsc) {
1342         gboolean child_active = rsc->fns->active(rsc, all);
1343 
1344         if (child_active && !all) {
1345             return TRUE;
1346         } else if (!child_active && all) {
1347             return FALSE;
1348         }
1349     }
1350     return -1;
1351 }
1352 
1353 gboolean
1354 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1355 {
1356     pe__bundle_variant_data_t *bundle_data = NULL;
1357     GList *iter = NULL;
1358 
1359     get_bundle_variant_data(bundle_data, rsc);
1360     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1361         pcmk__bundle_replica_t *replica = iter->data;
1362         int rsc_active;
1363 
1364         rsc_active = replica_resource_active(replica->ip, all);
1365         if (rsc_active >= 0) {
1366             return (gboolean) rsc_active;
1367         }
1368 
1369         rsc_active = replica_resource_active(replica->child, all);
1370         if (rsc_active >= 0) {
1371             return (gboolean) rsc_active;
1372         }
1373 
1374         rsc_active = replica_resource_active(replica->container, all);
1375         if (rsc_active >= 0) {
1376             return (gboolean) rsc_active;
1377         }
1378 
1379         rsc_active = replica_resource_active(replica->remote, all);
1380         if (rsc_active >= 0) {
1381             return (gboolean) rsc_active;
1382         }
1383     }
1384 
1385     /* If "all" is TRUE, we've already checked that no resources were inactive,
1386      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1387      * so return FALSE.
1388      */
1389     return all;
1390 }
1391 
1392 /*!
1393  * \internal
1394  * \brief Find the bundle replica corresponding to a given node
1395  *
1396  * \param[in] bundle  Top-level bundle resource
1397  * \param[in] node    Node to search for
1398  *
1399  * \return Bundle replica if found, NULL otherwise
1400  */
1401 pcmk_resource_t *
1402 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
     /*  */
1403 {
1404     pe__bundle_variant_data_t *bundle_data = NULL;
1405 
1406     pcmk__assert((bundle != NULL) && (node != NULL));
1407 
1408     get_bundle_variant_data(bundle_data, bundle);
1409     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1410          gIter = gIter->next) {
1411         pcmk__bundle_replica_t *replica = gIter->data;
1412 
1413         pcmk__assert((replica != NULL) && (replica->node != NULL));
1414         if (pcmk__same_node(replica->node, node)) {
1415             return replica->child;
1416         }
1417     }
1418     return NULL;
1419 }
1420 
1421 /*!
1422  * \internal
1423  * \deprecated This function will be removed in a future release
1424  */
1425 static void
1426 print_rsc_in_list(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1427                   void *print_data)
1428 {
1429     if (rsc != NULL) {
1430         if (options & pe_print_html) {
1431             status_print("<li>");
1432         }
1433         rsc->fns->print(rsc, pre_text, options, print_data);
1434         if (options & pe_print_html) {
1435             status_print("</li>\n");
1436         }
1437     }
1438 }
1439 
1440 /*!
1441  * \internal
1442  * \deprecated This function will be removed in a future release
1443  */
1444 static void
1445 bundle_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1446                  void *print_data)
1447 {
1448     pe__bundle_variant_data_t *bundle_data = NULL;
1449     char *child_text = NULL;
1450     CRM_CHECK(rsc != NULL, return);
1451 
1452     if (pre_text == NULL) {
1453         pre_text = "";
1454     }
1455     child_text = crm_strdup_printf("%s        ", pre_text);
1456 
1457     get_bundle_variant_data(bundle_data, rsc);
1458 
1459     status_print("%s<bundle ", pre_text);
1460     status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
1461     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1462     status_print("image=\"%s\" ", bundle_data->image);
1463     status_print("unique=\"%s\" ",
1464                  pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
1465     status_print("managed=\"%s\" ",
1466                  pcmk__flag_text(rsc->flags, pcmk_rsc_managed));
1467     status_print("failed=\"%s\" ",
1468                  pcmk__flag_text(rsc->flags, pcmk_rsc_failed));
1469     status_print(">\n");
1470 
1471     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1472          gIter = gIter->next) {
1473         pcmk__bundle_replica_t *replica = gIter->data;
1474 
1475         pcmk__assert(replica);
1476         status_print("%s    <replica " PCMK_XA_ID "=\"%d\">\n",
1477                      pre_text, replica->offset);
1478         print_rsc_in_list(replica->ip, child_text, options, print_data);
1479         print_rsc_in_list(replica->child, child_text, options, print_data);
1480         print_rsc_in_list(replica->container, child_text, options, print_data);
1481         print_rsc_in_list(replica->remote, child_text, options, print_data);
1482         status_print("%s    </replica>\n", pre_text);
1483     }
1484     status_print("%s</bundle>\n", pre_text);
1485     free(child_text);
1486 }
1487 
1488 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1489                   "GList *")
1490 int
1491 pe__bundle_xml(pcmk__output_t *out, va_list args)
1492 {
1493     uint32_t show_opts = va_arg(args, uint32_t);
1494     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1495     GList *only_node = va_arg(args, GList *);
1496     GList *only_rsc = va_arg(args, GList *);
1497 
1498     pe__bundle_variant_data_t *bundle_data = NULL;
1499     int rc = pcmk_rc_no_output;
1500     gboolean printed_header = FALSE;
1501     gboolean print_everything = TRUE;
1502 
1503     const char *desc = NULL;
1504 
1505     pcmk__assert(rsc != NULL);
1506 
1507     get_bundle_variant_data(bundle_data, rsc);
1508 
1509     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1510         return rc;
1511     }
1512 
1513     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1514 
1515     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1516          gIter = gIter->next) {
1517         pcmk__bundle_replica_t *replica = gIter->data;
1518         char *id = NULL;
1519         gboolean print_ip, print_child, print_ctnr, print_remote;
1520 
1521         pcmk__assert(replica);
1522 
1523         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1524             continue;
1525         }
1526 
1527         print_ip = replica->ip != NULL &&
1528                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1529         print_child = replica->child != NULL &&
1530                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1531         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1532         print_remote = replica->remote != NULL &&
1533                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1534 
1535         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1536             continue;
1537         }
1538 
1539         if (!printed_header) {
1540             const char *type = container_agent_str(bundle_data->agent_type);
1541             const char *unique = pcmk__flag_text(rsc->flags, pcmk_rsc_unique);
1542             const char *maintenance = pcmk__flag_text(rsc->flags,
1543                                                       pcmk_rsc_maintenance);
1544             const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
1545             const char *failed = pcmk__flag_text(rsc->flags, pcmk_rsc_failed);
1546 
1547             printed_header = TRUE;
1548 
1549             desc = pe__resource_description(rsc, show_opts);
1550 
1551             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
1552                                           PCMK_XA_ID, rsc->id,
1553                                           PCMK_XA_TYPE, type,
1554                                           PCMK_XA_IMAGE, bundle_data->image,
1555                                           PCMK_XA_UNIQUE, unique,
1556                                           PCMK_XA_MAINTENANCE, maintenance,
1557                                           PCMK_XA_MANAGED, managed,
1558                                           PCMK_XA_FAILED, failed,
1559                                           PCMK_XA_DESCRIPTION, desc,
1560                                           NULL);
1561             pcmk__assert(rc == pcmk_rc_ok);
1562         }
1563 
1564         id = pcmk__itoa(replica->offset);
1565         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
1566                                       PCMK_XA_ID, id,
1567                                       NULL);
1568         free(id);
1569         pcmk__assert(rc == pcmk_rc_ok);
1570 
1571         if (print_ip) {
1572             out->message(out, (const char *) replica->ip->xml->name, show_opts,
1573                          replica->ip, only_node, only_rsc);
1574         }
1575 
1576         if (print_child) {
1577             out->message(out, (const char *) replica->child->xml->name,
1578                          show_opts, replica->child, only_node, only_rsc);
1579         }
1580 
1581         if (print_ctnr) {
1582             out->message(out, (const char *) replica->container->xml->name,
1583                          show_opts, replica->container, only_node, only_rsc);
1584         }
1585 
1586         if (print_remote) {
1587             out->message(out, (const char *) replica->remote->xml->name,
1588                          show_opts, replica->remote, only_node, only_rsc);
1589         }
1590 
1591         pcmk__output_xml_pop_parent(out); // replica
1592     }
1593 
1594     if (printed_header) {
1595         pcmk__output_xml_pop_parent(out); // bundle
1596     }
1597 
1598     return rc;
1599 }
1600 
1601 static void
1602 pe__bundle_replica_output_html(pcmk__output_t *out,
     /*  */
1603                                pcmk__bundle_replica_t *replica,
1604                                pcmk_node_t *node, uint32_t show_opts)
1605 {
1606     pcmk_resource_t *rsc = replica->child;
1607 
1608     int offset = 0;
1609     char buffer[LINE_MAX];
1610 
1611     if(rsc == NULL) {
1612         rsc = replica->container;
1613     }
1614 
1615     if (replica->remote) {
1616         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1617                            rsc_printable_id(replica->remote));
1618     } else {
1619         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1620                            rsc_printable_id(replica->container));
1621     }
1622     if (replica->ipaddr) {
1623         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1624                            replica->ipaddr);
1625     }
1626 
1627     pe__common_output_html(out, rsc, buffer, node, show_opts);
1628 }
1629 
1630 /*!
1631  * \internal
1632  * \brief Get a string describing a resource's unmanaged state or lack thereof
1633  *
1634  * \param[in] rsc  Resource to describe
1635  *
1636  * \return A string indicating that a resource is in maintenance mode or
1637  *         otherwise unmanaged, or an empty string otherwise
1638  */
1639 static const char *
1640 get_unmanaged_str(const pcmk_resource_t *rsc)
     /*  */
1641 {
1642     if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
1643         return " (maintenance)";
1644     }
1645     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
1646         return " (unmanaged)";
1647     }
1648     return "";
1649 }
1650 
1651 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1652                   "GList *")
1653 int
1654 pe__bundle_html(pcmk__output_t *out, va_list args)
1655 {
1656     uint32_t show_opts = va_arg(args, uint32_t);
1657     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1658     GList *only_node = va_arg(args, GList *);
1659     GList *only_rsc = va_arg(args, GList *);
1660 
1661     const char *desc = NULL;
1662     pe__bundle_variant_data_t *bundle_data = NULL;
1663     int rc = pcmk_rc_no_output;
1664     gboolean print_everything = TRUE;
1665 
1666     pcmk__assert(rsc != NULL);
1667 
1668     get_bundle_variant_data(bundle_data, rsc);
1669 
1670     desc = pe__resource_description(rsc, show_opts);
1671 
1672     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1673         return rc;
1674     }
1675 
1676     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1677 
1678     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1679          gIter = gIter->next) {
1680         pcmk__bundle_replica_t *replica = gIter->data;
1681         gboolean print_ip, print_child, print_ctnr, print_remote;
1682 
1683         pcmk__assert(replica);
1684 
1685         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1686             continue;
1687         }
1688 
1689         print_ip = replica->ip != NULL &&
1690                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1691         print_child = replica->child != NULL &&
1692                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1693         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1694         print_remote = replica->remote != NULL &&
1695                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1696 
1697         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1698             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1699             /* The text output messages used below require pe_print_implicit to
1700              * be set to do anything.
1701              */
1702             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1703 
1704             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1705                                      (bundle_data->nreplicas > 1)? " set" : "",
1706                                      rsc->id, bundle_data->image,
1707                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1708                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1709                                      get_unmanaged_str(rsc));
1710 
1711             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1712                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1713             }
1714 
1715             if (print_ip) {
1716                 out->message(out, (const char *) replica->ip->xml->name,
1717                              new_show_opts, replica->ip, only_node, only_rsc);
1718             }
1719 
1720             if (print_child) {
1721                 out->message(out, (const char *) replica->child->xml->name,
1722                              new_show_opts, replica->child, only_node,
1723                              only_rsc);
1724             }
1725 
1726             if (print_ctnr) {
1727                 out->message(out, (const char *) replica->container->xml->name,
1728                              new_show_opts, replica->container, only_node,
1729                              only_rsc);
1730             }
1731 
1732             if (print_remote) {
1733                 out->message(out, (const char *) replica->remote->xml->name,
1734                              new_show_opts, replica->remote, only_node,
1735                              only_rsc);
1736             }
1737 
1738             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1739                 out->end_list(out);
1740             }
1741         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1742             continue;
1743         } else {
1744             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1745                                      (bundle_data->nreplicas > 1)? " set" : "",
1746                                      rsc->id, bundle_data->image,
1747                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1748                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1749                                      get_unmanaged_str(rsc));
1750 
1751             pe__bundle_replica_output_html(out, replica,
1752                                            pcmk__current_node(replica->container),
1753                                            show_opts);
1754         }
1755     }
1756 
1757     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1758     return rc;
1759 }
1760 
1761 static void
1762 pe__bundle_replica_output_text(pcmk__output_t *out,
     /*  */
1763                                pcmk__bundle_replica_t *replica,
1764                                pcmk_node_t *node, uint32_t show_opts)
1765 {
1766     const pcmk_resource_t *rsc = replica->child;
1767 
1768     int offset = 0;
1769     char buffer[LINE_MAX];
1770 
1771     if(rsc == NULL) {
1772         rsc = replica->container;
1773     }
1774 
1775     if (replica->remote) {
1776         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1777                            rsc_printable_id(replica->remote));
1778     } else {
1779         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1780                            rsc_printable_id(replica->container));
1781     }
1782     if (replica->ipaddr) {
1783         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1784                            replica->ipaddr);
1785     }
1786 
1787     pe__common_output_text(out, rsc, buffer, node, show_opts);
1788 }
1789 
1790 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1791                   "GList *")
1792 int
1793 pe__bundle_text(pcmk__output_t *out, va_list args)
1794 {
1795     uint32_t show_opts = va_arg(args, uint32_t);
1796     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1797     GList *only_node = va_arg(args, GList *);
1798     GList *only_rsc = va_arg(args, GList *);
1799 
1800     const char *desc = NULL;
1801     pe__bundle_variant_data_t *bundle_data = NULL;
1802     int rc = pcmk_rc_no_output;
1803     gboolean print_everything = TRUE;
1804 
1805     desc = pe__resource_description(rsc, show_opts);
1806     
1807     get_bundle_variant_data(bundle_data, rsc);
1808 
1809     pcmk__assert(rsc != NULL);
1810 
1811     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1812         return rc;
1813     }
1814 
1815     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1816 
1817     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1818          gIter = gIter->next) {
1819         pcmk__bundle_replica_t *replica = gIter->data;
1820         gboolean print_ip, print_child, print_ctnr, print_remote;
1821 
1822         pcmk__assert(replica);
1823 
1824         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1825             continue;
1826         }
1827 
1828         print_ip = replica->ip != NULL &&
1829                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1830         print_child = replica->child != NULL &&
1831                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1832         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1833         print_remote = replica->remote != NULL &&
1834                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1835 
1836         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1837             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1838             /* The text output messages used below require pe_print_implicit to
1839              * be set to do anything.
1840              */
1841             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1842 
1843             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1844                                      (bundle_data->nreplicas > 1)? " set" : "",
1845                                      rsc->id, bundle_data->image,
1846                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1847                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1848                                      get_unmanaged_str(rsc));
1849 
1850             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1851                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1852             }
1853 
1854             out->begin_list(out, NULL, NULL, NULL);
1855 
1856             if (print_ip) {
1857                 out->message(out, (const char *) replica->ip->xml->name,
1858                              new_show_opts, replica->ip, only_node, only_rsc);
1859             }
1860 
1861             if (print_child) {
1862                 out->message(out, (const char *) replica->child->xml->name,
1863                              new_show_opts, replica->child, only_node,
1864                              only_rsc);
1865             }
1866 
1867             if (print_ctnr) {
1868                 out->message(out, (const char *) replica->container->xml->name,
1869                              new_show_opts, replica->container, only_node,
1870                              only_rsc);
1871             }
1872 
1873             if (print_remote) {
1874                 out->message(out, (const char *) replica->remote->xml->name,
1875                              new_show_opts, replica->remote, only_node,
1876                              only_rsc);
1877             }
1878 
1879             out->end_list(out);
1880         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1881             continue;
1882         } else {
1883             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1884                                      (bundle_data->nreplicas > 1)? " set" : "",
1885                                      rsc->id, bundle_data->image,
1886                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1887                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1888                                      get_unmanaged_str(rsc));
1889 
1890             pe__bundle_replica_output_text(out, replica,
1891                                            pcmk__current_node(replica->container),
1892                                            show_opts);
1893         }
1894     }
1895 
1896     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1897     return rc;
1898 }
1899 
1900 /*!
1901  * \internal
1902  * \deprecated This function will be removed in a future release
1903  */
1904 static void
1905 print_bundle_replica(pcmk__bundle_replica_t *replica, const char *pre_text,
     /*  */
1906                      long options, void *print_data)
1907 {
1908     pcmk_node_t *node = NULL;
1909     pcmk_resource_t *rsc = replica->child;
1910 
1911     int offset = 0;
1912     char buffer[LINE_MAX];
1913 
1914     if(rsc == NULL) {
1915         rsc = replica->container;
1916     }
1917 
1918     if (replica->remote) {
1919         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1920                            rsc_printable_id(replica->remote));
1921     } else {
1922         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1923                            rsc_printable_id(replica->container));
1924     }
1925     if (replica->ipaddr) {
1926         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1927                            replica->ipaddr);
1928     }
1929 
1930     node = pcmk__current_node(replica->container);
1931     common_print(rsc, pre_text, buffer, node, options, print_data);
1932 }
1933 
1934 /*!
1935  * \internal
1936  * \deprecated This function will be removed in a future release
1937  */
1938 void
1939 pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1940                  void *print_data)
1941 {
1942     pe__bundle_variant_data_t *bundle_data = NULL;
1943     char *child_text = NULL;
1944     CRM_CHECK(rsc != NULL, return);
1945 
1946     if (options & pe_print_xml) {
1947         bundle_print_xml(rsc, pre_text, options, print_data);
1948         return;
1949     }
1950 
1951     get_bundle_variant_data(bundle_data, rsc);
1952 
1953     if (pre_text == NULL) {
1954         pre_text = " ";
1955     }
1956 
1957     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1958                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1959                  rsc->id, bundle_data->image,
1960                  pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1961                  pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
1962     if (options & pe_print_html) {
1963         status_print("<br />\n<ul>\n");
1964     }
1965 
1966 
1967     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1968          gIter = gIter->next) {
1969         pcmk__bundle_replica_t *replica = gIter->data;
1970 
1971         pcmk__assert(replica);
1972         if (options & pe_print_html) {
1973             status_print("<li>");
1974         }
1975 
1976         if (pcmk_is_set(options, pe_print_implicit)) {
1977             child_text = crm_strdup_printf("     %s", pre_text);
1978             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1979                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1980             }
1981             if (options & pe_print_html) {
1982                 status_print("<br />\n<ul>\n");
1983             }
1984             print_rsc_in_list(replica->ip, child_text, options, print_data);
1985             print_rsc_in_list(replica->container, child_text, options, print_data);
1986             print_rsc_in_list(replica->remote, child_text, options, print_data);
1987             print_rsc_in_list(replica->child, child_text, options, print_data);
1988             if (options & pe_print_html) {
1989                 status_print("</ul>\n");
1990             }
1991         } else {
1992             child_text = crm_strdup_printf("%s  ", pre_text);
1993             print_bundle_replica(replica, child_text, options, print_data);
1994         }
1995         free(child_text);
1996 
1997         if (options & pe_print_html) {
1998             status_print("</li>\n");
1999         }
2000     }
2001     if (options & pe_print_html) {
2002         status_print("</ul>\n");
2003     }
2004 }
2005 
2006 static void
2007 free_bundle_replica(pcmk__bundle_replica_t *replica)
     /*  */
2008 {
2009     if (replica == NULL) {
2010         return;
2011     }
2012 
2013     if (replica->node) {
2014         free(replica->node);
2015         replica->node = NULL;
2016     }
2017 
2018     if (replica->ip) {
2019         free_xml(replica->ip->xml);
2020         replica->ip->xml = NULL;
2021         replica->ip->fns->free(replica->ip);
2022         replica->ip = NULL;
2023     }
2024     if (replica->container) {
2025         free_xml(replica->container->xml);
2026         replica->container->xml = NULL;
2027         replica->container->fns->free(replica->container);
2028         replica->container = NULL;
2029     }
2030     if (replica->remote) {
2031         free_xml(replica->remote->xml);
2032         replica->remote->xml = NULL;
2033         replica->remote->fns->free(replica->remote);
2034         replica->remote = NULL;
2035     }
2036     free(replica->ipaddr);
2037     free(replica);
2038 }
2039 
2040 void
2041 pe__free_bundle(pcmk_resource_t *rsc)
     /*  */
2042 {
2043     pe__bundle_variant_data_t *bundle_data = NULL;
2044     CRM_CHECK(rsc != NULL, return);
2045 
2046     get_bundle_variant_data(bundle_data, rsc);
2047     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
2048 
2049     free(bundle_data->prefix);
2050     free(bundle_data->image);
2051     free(bundle_data->control_port);
2052     free(bundle_data->host_network);
2053     free(bundle_data->host_netmask);
2054     free(bundle_data->ip_range_start);
2055     free(bundle_data->container_network);
2056     free(bundle_data->launcher_options);
2057     free(bundle_data->container_command);
2058     g_free(bundle_data->container_host_options);
2059 
2060     g_list_free_full(bundle_data->replicas,
2061                      (GDestroyNotify) free_bundle_replica);
2062     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
2063     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
2064     g_list_free(rsc->children);
2065 
2066     if(bundle_data->child) {
2067         free_xml(bundle_data->child->xml);
2068         bundle_data->child->xml = NULL;
2069         bundle_data->child->fns->free(bundle_data->child);
2070     }
2071     common_free(rsc);
2072 }
2073 
2074 enum rsc_role_e
2075 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
     /*  */
2076 {
2077     enum rsc_role_e container_role = pcmk_role_unknown;
2078     return container_role;
2079 }
2080 
2081 /*!
2082  * \brief Get the number of configured replicas in a bundle
2083  *
2084  * \param[in] rsc  Bundle resource
2085  *
2086  * \return Number of configured replicas, or 0 on error
2087  */
2088 int
2089 pe_bundle_replicas(const pcmk_resource_t *rsc)
     /*  */
2090 {
2091     if (pcmk__is_bundle(rsc)) {
2092         pe__bundle_variant_data_t *bundle_data = NULL;
2093 
2094         get_bundle_variant_data(bundle_data, rsc);
2095         return bundle_data->nreplicas;
2096     }
2097     return 0;
2098 }
2099 
2100 void
2101 pe__count_bundle(pcmk_resource_t *rsc)
     /*  */
2102 {
2103     pe__bundle_variant_data_t *bundle_data = NULL;
2104 
2105     get_bundle_variant_data(bundle_data, rsc);
2106     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2107         pcmk__bundle_replica_t *replica = item->data;
2108 
2109         if (replica->ip) {
2110             replica->ip->fns->count(replica->ip);
2111         }
2112         if (replica->child) {
2113             replica->child->fns->count(replica->child);
2114         }
2115         if (replica->container) {
2116             replica->container->fns->count(replica->container);
2117         }
2118         if (replica->remote) {
2119             replica->remote->fns->count(replica->remote);
2120         }
2121     }
2122 }
2123 
2124 gboolean
2125 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
     /*  */
2126                        gboolean check_parent)
2127 {
2128     gboolean passes = FALSE;
2129     pe__bundle_variant_data_t *bundle_data = NULL;
2130 
2131     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
2132         passes = TRUE;
2133     } else {
2134         get_bundle_variant_data(bundle_data, rsc);
2135 
2136         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2137             pcmk__bundle_replica_t *replica = gIter->data;
2138 
2139             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2140                 passes = TRUE;
2141                 break;
2142             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2143                 passes = TRUE;
2144                 break;
2145             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2146                 passes = TRUE;
2147                 break;
2148             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2149                 passes = TRUE;
2150                 break;
2151             }
2152         }
2153     }
2154 
2155     return !passes;
2156 }
2157 
2158 /*!
2159  * \internal
2160  * \brief Get a list of a bundle's containers
2161  *
2162  * \param[in] bundle  Bundle resource
2163  *
2164  * \return Newly created list of \p bundle's containers
2165  * \note It is the caller's responsibility to free the result with
2166  *       g_list_free().
2167  */
2168 GList *
2169 pe__bundle_containers(const pcmk_resource_t *bundle)
     /*  */
2170 {
2171     GList *containers = NULL;
2172     const pe__bundle_variant_data_t *data = NULL;
2173 
2174     get_bundle_variant_data(data, bundle);
2175     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
2176         pcmk__bundle_replica_t *replica = iter->data;
2177 
2178         containers = g_list_append(containers, replica->container);
2179     }
2180     return containers;
2181 }
2182 
2183 // Bundle implementation of pcmk_rsc_methods_t:active_node()
2184 pcmk_node_t *
2185 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
     /*  */
2186                        unsigned int *count_clean)
2187 {
2188     pcmk_node_t *active = NULL;
2189     pcmk_node_t *node = NULL;
2190     pcmk_resource_t *container = NULL;
2191     GList *containers = NULL;
2192     GList *iter = NULL;
2193     GHashTable *nodes = NULL;
2194     const pe__bundle_variant_data_t *data = NULL;
2195 
2196     if (count_all != NULL) {
2197         *count_all = 0;
2198     }
2199     if (count_clean != NULL) {
2200         *count_clean = 0;
2201     }
2202     if (rsc == NULL) {
2203         return NULL;
2204     }
2205 
2206     /* For the purposes of this method, we only care about where the bundle's
2207      * containers are active, so build a list of active containers.
2208      */
2209     get_bundle_variant_data(data, rsc);
2210     for (iter = data->replicas; iter != NULL; iter = iter->next) {
2211         pcmk__bundle_replica_t *replica = iter->data;
2212 
2213         if (replica->container->running_on != NULL) {
2214             containers = g_list_append(containers, replica->container);
2215         }
2216     }
2217     if (containers == NULL) {
2218         return NULL;
2219     }
2220 
2221     /* If the bundle has only a single active container, just use that
2222      * container's method. If live migration is ever supported for bundle
2223      * containers, this will allow us to prefer the migration source when there
2224      * is only one container and it is migrating. For now, this just lets us
2225      * avoid creating the nodes table.
2226      */
2227     if (pcmk__list_of_1(containers)) {
2228         container = containers->data;
2229         node = container->fns->active_node(container, count_all, count_clean);
2230         g_list_free(containers);
2231         return node;
2232     }
2233 
2234     // Add all containers' active nodes to a hash table (for uniqueness)
2235     nodes = g_hash_table_new(NULL, NULL);
2236     for (iter = containers; iter != NULL; iter = iter->next) {
2237         container = iter->data;
2238 
2239         for (GList *node_iter = container->running_on; node_iter != NULL;
2240              node_iter = node_iter->next) {
2241             node = node_iter->data;
2242 
2243             // If insert returns true, we haven't counted this node yet
2244             if (g_hash_table_insert(nodes, (gpointer) node->details,
2245                                     (gpointer) node)
2246                 && !pe__count_active_node(rsc, node, &active, count_all,
2247                                           count_clean)) {
2248                 goto done;
2249             }
2250         }
2251     }
2252 
2253 done:
2254     g_list_free(containers);
2255     g_hash_table_destroy(nodes);
2256     return active;
2257 }
2258 
2259 /*!
2260  * \internal
2261  * \brief Get maximum bundle resource instances per node
2262  *
2263  * \param[in] rsc  Bundle resource to check
2264  *
2265  * \return Maximum number of \p rsc instances that can be active on one node
2266  */
2267 unsigned int
2268 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
     /*  */
2269 {
2270     pe__bundle_variant_data_t *bundle_data = NULL;
2271 
2272     get_bundle_variant_data(bundle_data, rsc);
2273     pcmk__assert(bundle_data->nreplicas_per_host >= 0);
2274     return (unsigned int) bundle_data->nreplicas_per_host;
2275 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
