lib/pengine/bundle.c

/* */
This source file includes following definitions.
pe__bundle_max
pe__bundled_resource
pe__get_rsc_in_container
pe__node_is_bundle_instance
pe__first_container
pe__foreach_bundle_replica
pe__foreach_const_bundle_replica
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
container_agent_str
create_container_resource
disallow_node
create_remote_resource
create_replica_resources
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
get_unmanaged_str
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
pe__bundle_containers
pe__bundle_active_node
pe__bundle_max_per_node
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/common/xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 enum pe__bundle_mount_flags {
  24     pe__bundle_mount_none       = 0x00,
  25 
  26     // mount instance-specific subdirectory rather than source directly
  27     pe__bundle_mount_subdir     = 0x01
  28 };
  29 
  30 typedef struct {
  31     char *source;
  32     char *target;
  33     char *options;
  34     uint32_t flags; // bitmask of pe__bundle_mount_flags
  35 } pe__bundle_mount_t;
  36 
  37 typedef struct {
  38     char *source;
  39     char *target;
  40 } pe__bundle_port_t;
  41 
  42 enum pe__container_agent {
  43     PE__CONTAINER_AGENT_UNKNOWN,
  44     PE__CONTAINER_AGENT_DOCKER,
  45     PE__CONTAINER_AGENT_RKT,
  46     PE__CONTAINER_AGENT_PODMAN,
  47 };
  48 
  49 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
  50 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
  51 #define PE__CONTAINER_AGENT_RKT_S     "rkt"
  52 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
  53 
  54 typedef struct pe__bundle_variant_data_s {
  55         int promoted_max;
  56         int nreplicas;
  57         int nreplicas_per_host;
  58         char *prefix;
  59         char *image;
  60         const char *ip_last;
  61         char *host_network;
  62         char *host_netmask;
  63         char *control_port;
  64         char *container_network;
  65         char *ip_range_start;
  66         gboolean add_host;
  67         gchar *container_host_options;
  68         char *container_command;
  69         char *launcher_options;
  70         const char *attribute_target;
  71 
  72         pcmk_resource_t *child;
  73 
  74         GList *replicas;    // pcmk__bundle_replica_t *
  75         GList *ports;       // pe__bundle_port_t *
  76         GList *mounts;      // pe__bundle_mount_t *
  77 
  78         enum pe__container_agent agent_type;
  79 } pe__bundle_variant_data_t;
  80 
  81 #define get_bundle_variant_data(data, rsc)                              \
  82     CRM_ASSERT(pcmk__is_bundle(rsc) && (rsc->variant_opaque != NULL));  \
  83     data = (pe__bundle_variant_data_t *) rsc->variant_opaque;
  84 
  85 /*!
  86  * \internal
  87  * \brief Get maximum number of bundle replicas allowed to run
  88  *
  89  * \param[in] rsc  Bundle or bundled resource to check
  90  *
  91  * \return Maximum replicas for bundle corresponding to \p rsc
  92  */
  93 int
  94 pe__bundle_max(const pcmk_resource_t *rsc)
     /*  */
  95 {
  96     const pe__bundle_variant_data_t *bundle_data = NULL;
  97 
  98     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
  99     return bundle_data->nreplicas;
 100 }
 101 
 102 /*!
 103  * \internal
 104  * \brief Get the resource inside a bundle
 105  *
 106  * \param[in] bundle  Bundle to check
 107  *
 108  * \return Resource inside \p bundle if any, otherwise NULL
 109  */
 110 pcmk_resource_t *
 111 pe__bundled_resource(const pcmk_resource_t *rsc)
     /*  */
 112 {
 113     const pe__bundle_variant_data_t *bundle_data = NULL;
 114 
 115     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 116     return bundle_data->child;
 117 }
 118 
 119 /*!
 120  * \internal
 121  * \brief Get containerized resource corresponding to a given bundle container
 122  *
 123  * \param[in] instance  Collective instance that might be a bundle container
 124  *
 125  * \return Bundled resource instance inside \p instance if it is a bundle
 126  *         container instance, otherwise NULL
 127  */
 128 const pcmk_resource_t *
 129 pe__get_rsc_in_container(const pcmk_resource_t *instance)
     /*  */
 130 {
 131     const pe__bundle_variant_data_t *data = NULL;
 132     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 133 
 134     if (!pcmk__is_bundle(top)) {
 135         return NULL;
 136     }
 137     get_bundle_variant_data(data, top);
 138 
 139     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 140         const pcmk__bundle_replica_t *replica = iter->data;
 141 
 142         if (instance == replica->container) {
 143             return replica->child;
 144         }
 145     }
 146     return NULL;
 147 }
 148 
 149 /*!
 150  * \internal
 151  * \brief Check whether a given node is created by a bundle
 152  *
 153  * \param[in] bundle  Bundle resource to check
 154  * \param[in] node    Node to check
 155  *
 156  * \return true if \p node is an instance of \p bundle, otherwise false
 157  */
 158 bool
 159 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
     /*  */
 160                             const pcmk_node_t *node)
 161 {
 162     pe__bundle_variant_data_t *bundle_data = NULL;
 163 
 164     get_bundle_variant_data(bundle_data, bundle);
 165     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 166         pcmk__bundle_replica_t *replica = iter->data;
 167 
 168         if (pcmk__same_node(node, replica->node)) {
 169             return true;
 170         }
 171     }
 172     return false;
 173 }
 174 
 175 /*!
 176  * \internal
 177  * \brief Get the container of a bundle's first replica
 178  *
 179  * \param[in] bundle  Bundle resource to get container for
 180  *
 181  * \return Container resource from first replica of \p bundle if any,
 182  *         otherwise NULL
 183  */
 184 pcmk_resource_t *
 185 pe__first_container(const pcmk_resource_t *bundle)
     /*  */
 186 {
 187     const pe__bundle_variant_data_t *bundle_data = NULL;
 188     const pcmk__bundle_replica_t *replica = NULL;
 189 
 190     get_bundle_variant_data(bundle_data, bundle);
 191     if (bundle_data->replicas == NULL) {
 192         return NULL;
 193     }
 194     replica = bundle_data->replicas->data;
 195     return replica->container;
 196 }
 197 
 198 /*!
 199  * \internal
 200  * \brief Iterate over bundle replicas
 201  *
 202  * \param[in,out] bundle     Bundle to iterate over
 203  * \param[in]     fn         Function to call for each replica (its return value
 204  *                           indicates whether to continue iterating)
 205  * \param[in,out] user_data  Pointer to pass to \p fn
 206  */
 207 void
 208 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
     /*  */
 209                            bool (*fn)(pcmk__bundle_replica_t *, void *),
 210                            void *user_data)
 211 {
 212     const pe__bundle_variant_data_t *bundle_data = NULL;
 213 
 214     get_bundle_variant_data(bundle_data, bundle);
 215     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 216         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
 217             break;
 218         }
 219     }
 220 }
 221 
 222 /*!
 223  * \internal
 224  * \brief Iterate over const bundle replicas
 225  *
 226  * \param[in]     bundle     Bundle to iterate over
 227  * \param[in]     fn         Function to call for each replica (its return value
 228  *                           indicates whether to continue iterating)
 229  * \param[in,out] user_data  Pointer to pass to \p fn
 230  */
 231 void
 232 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
     /*  */
 233                                  bool (*fn)(const pcmk__bundle_replica_t *,
 234                                             void *),
 235                                  void *user_data)
 236 {
 237     const pe__bundle_variant_data_t *bundle_data = NULL;
 238 
 239     get_bundle_variant_data(bundle_data, bundle);
 240     for (const GList *iter = bundle_data->replicas; iter != NULL;
 241          iter = iter->next) {
 242 
 243         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
 244             break;
 245         }
 246     }
 247 }
 248 
 249 static char *
 250 next_ip(const char *last_ip)
     /*  */
 251 {
 252     unsigned int oct1 = 0;
 253     unsigned int oct2 = 0;
 254     unsigned int oct3 = 0;
 255     unsigned int oct4 = 0;
 256     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 257 
 258     if (rc != 4) {
 259         /*@ TODO check for IPv6 */
 260         return NULL;
 261 
 262     } else if (oct3 > 253) {
 263         return NULL;
 264 
 265     } else if (oct4 > 253) {
 266         ++oct3;
 267         oct4 = 1;
 268 
 269     } else {
 270         ++oct4;
 271     }
 272 
 273     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 274 }
 275 
 276 static void
 277 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
     /*  */
 278             GString *buffer)
 279 {
 280     if(data->ip_range_start == NULL) {
 281         return;
 282 
 283     } else if(data->ip_last) {
 284         replica->ipaddr = next_ip(data->ip_last);
 285 
 286     } else {
 287         replica->ipaddr = strdup(data->ip_range_start);
 288     }
 289 
 290     data->ip_last = replica->ipaddr;
 291     switch (data->agent_type) {
 292         case PE__CONTAINER_AGENT_DOCKER:
 293         case PE__CONTAINER_AGENT_PODMAN:
 294             if (data->add_host) {
 295                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 296                                        data->prefix, replica->offset,
 297                                        replica->ipaddr);
 298             } else {
 299                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 300                                        replica->ipaddr, data->prefix,
 301                                        replica->offset);
 302             }
 303             break;
 304 
 305         case PE__CONTAINER_AGENT_RKT:
 306             g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 307                                    replica->ipaddr, data->prefix,
 308                                    replica->offset);
 309             break;
 310 
 311         default: // PE__CONTAINER_AGENT_UNKNOWN
 312             break;
 313     }
 314 }
 315 
 316 static xmlNode *
 317 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 318 {
 319     xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 320 
 321     crm_xml_add(rsc, PCMK_XA_ID, name);
 322     crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
 323     crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
 324     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 325 
 326     return rsc;
 327 }
 328 
 329 /*!
 330  * \internal
 331  * \brief Check whether cluster can manage resource inside container
 332  *
 333  * \param[in,out] data  Container variant data
 334  *
 335  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 336  *
 337  * \note The resource is manageable if an IP range or control port has been
 338  *       specified. If a control port is used without an IP range, replicas per
 339  *       host must be 1.
 340  */
 341 static bool
 342 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 343 {
 344     if(data->ip_range_start) {
 345         return TRUE;
 346     }
 347     if(data->control_port) {
 348         if(data->nreplicas_per_host > 1) {
 349             pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
 350                              "requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
 351                              data->prefix);
 352             data->nreplicas_per_host = 1;
 353             // @TODO to be sure:
 354             // pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
 355         }
 356         return TRUE;
 357     }
 358     return FALSE;
 359 }
 360 
 361 static int
 362 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 363                    pcmk__bundle_replica_t *replica)
 364 {
 365     if(data->ip_range_start) {
 366         char *id = NULL;
 367         xmlNode *xml_ip = NULL;
 368         xmlNode *xml_obj = NULL;
 369 
 370         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 371         crm_xml_sanitize_id(id);
 372         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 373         free(id);
 374 
 375         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
 376         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 377                        data->prefix, replica->offset);
 378 
 379         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 380         if(data->host_network) {
 381             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 382         }
 383 
 384         if(data->host_netmask) {
 385             crm_create_nvpair_xml(xml_obj, NULL,
 386                                   "cidr_netmask", data->host_netmask);
 387 
 388         } else {
 389             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 390         }
 391 
 392         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
 393         crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
 394                           "60s", NULL);
 395 
 396         // TODO: Other ops? Timeouts and intervals from underlying resource?
 397 
 398         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 399                                 parent->cluster) != pcmk_rc_ok) {
 400             return pcmk_rc_unpack_error;
 401         }
 402 
 403         parent->children = g_list_append(parent->children, replica->ip);
 404     }
 405     return pcmk_rc_ok;
 406 }
 407 
 408 static const char*
 409 container_agent_str(enum pe__container_agent t)
     /*  */
 410 {
 411     switch (t) {
 412         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 413         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
 414         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 415         default: // PE__CONTAINER_AGENT_UNKNOWN
 416             break;
 417     }
 418     return PE__CONTAINER_AGENT_UNKNOWN_S;
 419 }
 420 
 421 static int
 422 create_container_resource(pcmk_resource_t *parent,
     /*  */
 423                           const pe__bundle_variant_data_t *data,
 424                           pcmk__bundle_replica_t *replica)
 425 {
 426     char *id = NULL;
 427     xmlNode *xml_container = NULL;
 428     xmlNode *xml_obj = NULL;
 429 
 430     // Agent-specific
 431     const char *hostname_opt = NULL;
 432     const char *env_opt = NULL;
 433     const char *agent_str = NULL;
 434     int volid = 0;  // rkt-only
 435 
 436     GString *buffer = NULL;
 437     GString *dbuffer = NULL;
 438 
 439     // Where syntax differences are drop-in replacements, set them now
 440     switch (data->agent_type) {
 441         case PE__CONTAINER_AGENT_DOCKER:
 442         case PE__CONTAINER_AGENT_PODMAN:
 443             hostname_opt = "-h ";
 444             env_opt = "-e ";
 445             break;
 446         case PE__CONTAINER_AGENT_RKT:
 447             hostname_opt = "--hostname=";
 448             env_opt = "--environment=";
 449             break;
 450         default:    // PE__CONTAINER_AGENT_UNKNOWN
 451             return pcmk_rc_unpack_error;
 452     }
 453     agent_str = container_agent_str(data->agent_type);
 454 
 455     buffer = g_string_sized_new(4096);
 456 
 457     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 458                            replica->offset);
 459     crm_xml_sanitize_id(id);
 460     xml_container = create_resource(id, "heartbeat", agent_str);
 461     free(id);
 462 
 463     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
 464     crm_xml_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 465 
 466     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 467     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
 468     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
 469     crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);
 470 
 471     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 472         g_string_append(buffer, " --restart=no");
 473     }
 474 
 475     /* Set a container hostname only if we have an IP to map it to. The user can
 476      * set -h or --uts=host themselves if they want a nicer name for logs, but
 477      * this makes applications happy who need their  hostname to match the IP
 478      * they bind to.
 479      */
 480     if (data->ip_range_start != NULL) {
 481         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 482                                replica->offset);
 483     }
 484     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 485 
 486     if (data->container_network != NULL) {
 487         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 488     }
 489 
 490     if (data->control_port != NULL) {
 491         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
 492                        data->control_port, NULL);
 493     } else {
 494         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
 495                                env_opt, DEFAULT_REMOTE_PORT);
 496     }
 497 
 498     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 499         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 500         char *source = NULL;
 501 
 502         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 503             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 504                                        replica->offset);
 505             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 506         }
 507 
 508         switch (data->agent_type) {
 509             case PE__CONTAINER_AGENT_DOCKER:
 510             case PE__CONTAINER_AGENT_PODMAN:
 511                 pcmk__g_strcat(buffer,
 512                                " -v ", pcmk__s(source, mount->source),
 513                                ":", mount->target, NULL);
 514 
 515                 if (mount->options != NULL) {
 516                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 517                 }
 518                 break;
 519             case PE__CONTAINER_AGENT_RKT:
 520                 g_string_append_printf(buffer,
 521                                        " --volume vol%d,kind=host,"
 522                                        "source=%s%s%s "
 523                                        "--mount volume=vol%d,target=%s",
 524                                        volid, pcmk__s(source, mount->source),
 525                                        (mount->options != NULL)? "," : "",
 526                                        pcmk__s(mount->options, ""),
 527                                        volid, mount->target);
 528                 volid++;
 529                 break;
 530             default:
 531                 break;
 532         }
 533         free(source);
 534     }
 535 
 536     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 537         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 538 
 539         switch (data->agent_type) {
 540             case PE__CONTAINER_AGENT_DOCKER:
 541             case PE__CONTAINER_AGENT_PODMAN:
 542                 if (replica->ipaddr != NULL) {
 543                     pcmk__g_strcat(buffer,
 544                                    " -p ", replica->ipaddr, ":", port->source,
 545                                    ":", port->target, NULL);
 546 
 547                 } else if (!pcmk__str_eq(data->container_network,
 548                                          PCMK_VALUE_HOST, pcmk__str_none)) {
 549                     // No need to do port mapping if net == host
 550                     pcmk__g_strcat(buffer,
 551                                    " -p ", port->source, ":", port->target,
 552                                    NULL);
 553                 }
 554                 break;
 555             case PE__CONTAINER_AGENT_RKT:
 556                 if (replica->ipaddr != NULL) {
 557                     pcmk__g_strcat(buffer,
 558                                    " --port=", port->target,
 559                                    ":", replica->ipaddr, ":", port->source,
 560                                    NULL);
 561                 } else {
 562                     pcmk__g_strcat(buffer,
 563                                    " --port=", port->target, ":", port->source,
 564                                    NULL);
 565                 }
 566                 break;
 567             default:
 568                 break;
 569         }
 570     }
 571 
 572     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 573      * it would cause restarts during rolling upgrades.
 574      *
 575      * In a previous version of the container resource creation logic, if
 576      * data->launcher_options is not NULL, we append
 577      * (" %s", data->launcher_options) even if data->launcher_options is an
 578      * empty string. Likewise for data->container_host_options. Using
 579      *
 580      *     pcmk__add_word(buffer, 0, data->launcher_options)
 581      *
 582      * removes that extra trailing space, causing a resource definition change.
 583      */
 584     if (data->launcher_options != NULL) {
 585         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 586     }
 587 
 588     if (data->container_host_options != NULL) {
 589         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 590     }
 591 
 592     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 593                           (const char *) buffer->str);
 594     g_string_free(buffer, TRUE);
 595 
 596     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 597                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 598     if (dbuffer != NULL) {
 599         g_string_free(dbuffer, TRUE);
 600     }
 601 
 602     if (replica->child != NULL) {
 603         if (data->container_command != NULL) {
 604             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 605                                   data->container_command);
 606         } else {
 607             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 608                                   SBIN_DIR "/pacemaker-remoted");
 609         }
 610 
 611         /* TODO: Allow users to specify their own?
 612          *
 613          * We just want to know if the container is alive; we'll monitor the
 614          * child independently.
 615          */
 616         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 617 #if 0
 618         /* @TODO Consider supporting the use case where we can start and stop
 619          * resources, but not proxy local commands (such as setting node
 620          * attributes), by running the local executor in stand-alone mode.
 621          * However, this would probably be better done via ACLs as with other
 622          * Pacemaker Remote nodes.
 623          */
 624     } else if ((child != NULL) && data->untrusted) {
 625         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 626                               CRM_DAEMON_DIR "/pacemaker-execd");
 627         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 628                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 629 #endif
 630     } else {
 631         if (data->container_command != NULL) {
 632             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 633                                   data->container_command);
 634         }
 635 
 636         /* TODO: Allow users to specify their own?
 637          *
 638          * We don't know what's in the container, so we just want to know if it
 639          * is alive.
 640          */
 641         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 642     }
 643 
 644     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
 645     crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
 646                       "60s", NULL);
 647 
 648     // TODO: Other ops? Timeouts and intervals from underlying resource?
 649     if (pe__unpack_resource(xml_container, &replica->container, parent,
 650                             parent->cluster) != pcmk_rc_ok) {
 651         return pcmk_rc_unpack_error;
 652     }
 653     pcmk__set_rsc_flags(replica->container, pcmk_rsc_replica_container);
 654     parent->children = g_list_append(parent->children, replica->container);
 655 
 656     return pcmk_rc_ok;
 657 }
 658 
 659 /*!
 660  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 661  *
 662  * \param[in,out] rsc    Resource to modify
 663  * \param[in]     uname  Name of node to ban
 664  */
 665 static void
 666 disallow_node(pcmk_resource_t *rsc, const char *uname)
     /*  */
 667 {
 668     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 669 
 670     if (match) {
 671         ((pcmk_node_t *) match)->weight = -PCMK_SCORE_INFINITY;
 672         ((pcmk_node_t *) match)->rsc_discover_mode = pcmk_probe_never;
 673     }
 674     if (rsc->children) {
 675         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 676     }
 677 }
 678 
 679 static int
 680 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 681                        pcmk__bundle_replica_t *replica)
 682 {
 683     if (replica->child && valid_network(data)) {
 684         GHashTableIter gIter;
 685         pcmk_node_t *node = NULL;
 686         xmlNode *xml_remote = NULL;
 687         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 688         char *port_s = NULL;
 689         const char *uname = NULL;
 690         const char *connect_name = NULL;
 691 
 692         if (pe_find_resource(parent->cluster->resources, id) != NULL) {
 693             free(id);
 694             // The biggest hammer we have
 695             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 696                                    replica->child->id, replica->offset);
 697             //@TODO return error instead of asserting?
 698             CRM_ASSERT(pe_find_resource(parent->cluster->resources,
 699                                         id) == NULL);
 700         }
 701 
 702         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 703          * connection does not have its own IP is a magic string that we use to
 704          * support nested remotes (i.e. a bundle running on a remote node).
 705          */
 706         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 707 
 708         if (data->control_port == NULL) {
 709             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 710         }
 711 
 712         /* This sets replica->container as replica->remote's container, which is
 713          * similar to what happens with guest nodes. This is how the scheduler
 714          * knows that the bundle node is fenced by recovering the container, and
 715          * that remote should be ordered relative to the container.
 716          */
 717         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 718                                           NULL, NULL, NULL,
 719                                           connect_name, (data->control_port?
 720                                           data->control_port : port_s));
 721         free(port_s);
 722 
 723         /* Abandon our created ID, and pull the copy from the XML, because we
 724          * need something that will get freed during scheduler data cleanup to
 725          * use as the node ID and uname.
 726          */
 727         free(id);
 728         id = NULL;
 729         uname = pcmk__xe_id(xml_remote);
 730 
 731         /* Ensure a node has been created for the guest (it may have already
 732          * been, if it has a permanent node attribute), and ensure its weight is
 733          * -INFINITY so no other resources can run on it.
 734          */
 735         node = pcmk_find_node(parent->cluster, uname);
 736         if (node == NULL) {
 737             node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
 738                                   PCMK_VALUE_MINUS_INFINITY, parent->cluster);
 739         } else {
 740             node->weight = -PCMK_SCORE_INFINITY;
 741         }
 742         node->rsc_discover_mode = pcmk_probe_never;
 743 
 744         /* unpack_remote_nodes() ensures that each remote node and guest node
 745          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
 746          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
 747          * obvious what nodes will be needed, so we do it just above.
 748          *
 749          * Worse, that means that the node may have been utilized while
 750          * unpacking other resources, without our weight correction. The most
 751          * likely place for this to happen is when pe__unpack_resource() calls
 752          * resource_location() to set a default score in symmetric clusters.
 753          * This adds a node *copy* to each resource's allowed nodes, and these
 754          * copies will have the wrong weight.
 755          *
 756          * As a hacky workaround, fix those copies here.
 757          *
 758          * @TODO Possible alternative: ensure bundles are unpacked before other
 759          * resources, so the weight is correct before any copies are made.
 760          */
 761         g_list_foreach(parent->cluster->resources, (GFunc) disallow_node,
 762                        (gpointer) uname);
 763 
 764         replica->node = pe__copy_node(node);
 765         replica->node->weight = 500;
 766         replica->node->rsc_discover_mode = pcmk_probe_exclusive;
 767 
 768         /* Ensure the node shows up as allowed and with the correct discovery set */
 769         if (replica->child->allowed_nodes != NULL) {
 770             g_hash_table_destroy(replica->child->allowed_nodes);
 771         }
 772         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 773         g_hash_table_insert(replica->child->allowed_nodes,
 774                             (gpointer) replica->node->details->id,
 775                             pe__copy_node(replica->node));
 776 
 777         {
 778             pcmk_node_t *copy = pe__copy_node(replica->node);
 779             copy->weight = -PCMK_SCORE_INFINITY;
 780             g_hash_table_insert(replica->child->parent->allowed_nodes,
 781                                 (gpointer) replica->node->details->id, copy);
 782         }
 783         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 784                                 parent->cluster) != pcmk_rc_ok) {
 785             return pcmk_rc_unpack_error;
 786         }
 787 
 788         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 789         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 790             if (pcmk__is_pacemaker_remote_node(node)) {
 791                 /* Remote resources can only run on 'normal' cluster node */
 792                 node->weight = -PCMK_SCORE_INFINITY;
 793             }
 794         }
 795 
 796         replica->node->details->remote_rsc = replica->remote;
 797 
 798         // Ensure pcmk__is_guest_or_bundle_node() functions correctly
 799         replica->remote->container = replica->container;
 800 
 801         /* A bundle's #kind is closer to "container" (guest node) than the
 802          * "remote" set by pe_create_node().
 803          */
 804         pcmk__insert_dup(replica->node->details->attrs,
 805                          CRM_ATTR_KIND, "container");
 806 
 807         /* One effect of this is that setup_container() will add
 808          * replica->remote to replica->container's fillers, which will make
 809          * pe__resource_contains_guest_node() true for replica->container.
 810          *
 811          * replica->child does NOT get added to replica->container's fillers.
 812          * The only noticeable effect if it did would be for its fail count to
 813          * be taken into account when checking replica->container's migration
 814          * threshold.
 815          */
 816         parent->children = g_list_append(parent->children, replica->remote);
 817     }
 818     return pcmk_rc_ok;
 819 }
 820 
 821 static int
 822 create_replica_resources(pcmk_resource_t *parent,
     /*  */
 823                          pe__bundle_variant_data_t *data,
 824                          pcmk__bundle_replica_t *replica)
 825 {
 826     int rc = pcmk_rc_ok;
 827 
 828     rc = create_container_resource(parent, data, replica);
 829     if (rc != pcmk_rc_ok) {
 830         return rc;
 831     }
 832 
 833     rc = create_ip_resource(parent, data, replica);
 834     if (rc != pcmk_rc_ok) {
 835         return rc;
 836     }
 837 
 838     rc = create_remote_resource(parent, data, replica);
 839     if (rc != pcmk_rc_ok) {
 840         return rc;
 841     }
 842 
 843     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 844         pcmk__insert_meta(replica->child, "external-ip", replica->ipaddr);
 845     }
 846 
 847     if (replica->remote != NULL) {
 848         /*
 849          * Allow the remote connection resource to be allocated to a
 850          * different node than the one on which the container is active.
 851          *
 852          * This makes it possible to have Pacemaker Remote nodes running
 853          * containers with pacemaker-remoted inside in order to start
 854          * services inside those containers.
 855          */
 856         pcmk__set_rsc_flags(replica->remote, pcmk_rsc_remote_nesting_allowed);
 857     }
 858     return rc;
 859 }
 860 
 861 static void
 862 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 863           const char *target, const char *options, uint32_t flags)
 864 {
 865     pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
 866                                                    sizeof(pe__bundle_mount_t));
 867 
 868     mount->source = pcmk__str_copy(source);
 869     mount->target = pcmk__str_copy(target);
 870     mount->options = pcmk__str_copy(options);
 871     mount->flags = flags;
 872     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 873 }
 874 
 875 static void
 876 mount_free(pe__bundle_mount_t *mount)
     /*  */
 877 {
 878     free(mount->source);
 879     free(mount->target);
 880     free(mount->options);
 881     free(mount);
 882 }
 883 
 884 static void
 885 port_free(pe__bundle_port_t *port)
     /*  */
 886 {
 887     free(port->source);
 888     free(port->target);
 889     free(port);
 890 }
 891 
 892 static pcmk__bundle_replica_t *
 893 replica_for_remote(pcmk_resource_t *remote)
     /*  */
 894 {
 895     pcmk_resource_t *top = remote;
 896     pe__bundle_variant_data_t *bundle_data = NULL;
 897 
 898     if (top == NULL) {
 899         return NULL;
 900     }
 901 
 902     while (top->parent != NULL) {
 903         top = top->parent;
 904     }
 905 
 906     get_bundle_variant_data(bundle_data, top);
 907     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 908          gIter = gIter->next) {
 909         pcmk__bundle_replica_t *replica = gIter->data;
 910 
 911         if (replica->remote == remote) {
 912             return replica;
 913         }
 914     }
 915     CRM_LOG_ASSERT(FALSE);
 916     return NULL;
 917 }
 918 
 919 bool
 920 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
     /*  */
 921 {
 922     const char *value;
 923     GHashTable *params = NULL;
 924 
 925     if (rsc == NULL) {
 926         return false;
 927     }
 928 
 929     // Use NULL node since pcmk__bundle_expand() uses that to set value
 930     params = pe_rsc_params(rsc, NULL, rsc->cluster);
 931     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 932 
 933     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 934            && xml_contains_remote_node(rsc->xml);
 935 }
 936 
 937 const char *
 938 pe__add_bundle_remote_name(pcmk_resource_t *rsc, xmlNode *xml,
     /*  */
 939                            const char *field)
 940 {
 941     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 942 
 943     pcmk_node_t *node = NULL;
 944     pcmk__bundle_replica_t *replica = NULL;
 945 
 946     if (!pe__bundle_needs_remote_name(rsc)) {
 947         return NULL;
 948     }
 949 
 950     replica = replica_for_remote(rsc);
 951     if (replica == NULL) {
 952         return NULL;
 953     }
 954 
 955     node = replica->container->allocated_to;
 956     if (node == NULL) {
 957         /* If it won't be running anywhere after the
 958          * transition, go with where it's running now.
 959          */
 960         node = pcmk__current_node(replica->container);
 961     }
 962 
 963     if(node == NULL) {
 964         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 965         return NULL;
 966     }
 967 
 968     crm_trace("Setting address for bundle connection %s to bundle host %s",
 969               rsc->id, pcmk__node_name(node));
 970     if(xml != NULL && field != NULL) {
 971         crm_xml_add(xml, field, node->details->uname);
 972     }
 973 
 974     return node->details->uname;
 975 }
 976 
 977 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 978         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 979                                    "Bundle mount", pcmk__xe_id(mount_xml),  \
 980                                    flags, (flags_to_set), #flags_to_set);   \
 981     } while (0)
 982 
 983 gboolean
 984 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
     /*  */
 985 {
 986     const char *value = NULL;
 987     xmlNode *xml_obj = NULL;
 988     const xmlNode *xml_child = NULL;
 989     xmlNode *xml_resource = NULL;
 990     pe__bundle_variant_data_t *bundle_data = NULL;
 991     bool need_log_mount = TRUE;
 992 
 993     CRM_ASSERT(rsc != NULL);
 994     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 995 
 996     bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
 997     rsc->variant_opaque = bundle_data;
 998     bundle_data->prefix = strdup(rsc->id);
 999 
1000     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_DOCKER, NULL, NULL);
1001     if (xml_obj != NULL) {
1002         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1003     } else {
1004         xml_obj = pcmk__xe_first_child(rsc->xml, PCMK__XE_RKT, NULL, NULL);
1005         if (xml_obj != NULL) {
1006             pcmk__warn_once(pcmk__wo_rkt,
1007                             "Support for " PCMK__XE_RKT " in bundles "
1008                             "(such as %s) is deprecated and will be "
1009                             "removed in a future release", rsc->id);
1010             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1011         } else {
1012             xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PODMAN, NULL,
1013                                            NULL);
1014             if (xml_obj != NULL) {
1015                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1016             } else {
1017                 return FALSE;
1018             }
1019         }
1020     }
1021 
1022     // Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
1023     value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
1024     if (value == NULL) {
1025         // @COMPAT deprecated since 2.0.0
1026         value = crm_element_value(xml_obj, PCMK__XA_PROMOTED_MAX_LEGACY);
1027 
1028         if (value != NULL) {
1029             pcmk__warn_once(pcmk__wo_bundle_master,
1030                             "Support for the " PCMK__XA_PROMOTED_MAX_LEGACY
1031                             " attribute (such as in %s) is deprecated and "
1032                             "will be removed in a future release. Use "
1033                             PCMK_XA_PROMOTED_MAX " instead.",
1034                             rsc->id);
1035         }
1036     }
1037     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1038 
1039     /* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
1040      * otherwise
1041      */
1042     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
1043     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1044         bundle_data->nreplicas = bundle_data->promoted_max;
1045     } else {
1046         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1047     }
1048 
1049     /*
1050      * Communication between containers on the same host via the
1051      * floating IPs only works if the container is started with:
1052      *   --userland-proxy=false --ip-masq=false
1053      */
1054     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
1055     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1056     if (bundle_data->nreplicas_per_host == 1) {
1057         pcmk__clear_rsc_flags(rsc, pcmk_rsc_unique);
1058     }
1059 
1060     bundle_data->container_command =
1061         crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
1062     bundle_data->launcher_options = crm_element_value_copy(xml_obj,
1063                                                            PCMK_XA_OPTIONS);
1064     bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
1065     bundle_data->container_network = crm_element_value_copy(xml_obj,
1066                                                             PCMK_XA_NETWORK);
1067 
1068     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_NETWORK, NULL, NULL);
1069     if(xml_obj) {
1070         bundle_data->ip_range_start =
1071             crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
1072         bundle_data->host_netmask =
1073             crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
1074         bundle_data->host_network =
1075             crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
1076         bundle_data->control_port =
1077             crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
1078         value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
1079         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1080             bundle_data->add_host = TRUE;
1081         }
1082 
1083         for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
1084                                               NULL, NULL);
1085              xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {
1086 
1087             pe__bundle_port_t *port =
1088                 pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1089 
1090             port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);
1091 
1092             if(port->source == NULL) {
1093                 port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
1094             } else {
1095                 port->target = crm_element_value_copy(xml_child,
1096                                                       PCMK_XA_INTERNAL_PORT);
1097             }
1098 
1099             if(port->source != NULL && strlen(port->source) > 0) {
1100                 if(port->target == NULL) {
1101                     port->target = strdup(port->source);
1102                 }
1103                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1104 
1105             } else {
1106                 pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
1107                                  pcmk__xe_id(xml_child));
1108                 port_free(port);
1109             }
1110         }
1111     }
1112 
1113     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_STORAGE, NULL, NULL);
1114     for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
1115                                           NULL, NULL);
1116          xml_child != NULL; xml_child = pcmk__xe_next_same(xml_child)) {
1117 
1118         const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
1119         const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
1120         const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
1121         int flags = pe__bundle_mount_none;
1122 
1123         if (source == NULL) {
1124             source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
1125             pe__set_bundle_mount_flags(xml_child, flags,
1126                                        pe__bundle_mount_subdir);
1127         }
1128 
1129         if (source && target) {
1130             mount_add(bundle_data, source, target, options, flags);
1131             if (strcmp(target, "/var/log") == 0) {
1132                 need_log_mount = FALSE;
1133             }
1134         } else {
1135             pcmk__config_err("Invalid mount directive %s",
1136                              pcmk__xe_id(xml_child));
1137         }
1138     }
1139 
1140     xml_obj = pcmk__xe_first_child(rsc->xml, PCMK_XE_PRIMITIVE, NULL, NULL);
1141     if (xml_obj && valid_network(bundle_data)) {
1142         char *value = NULL;
1143         xmlNode *xml_set = NULL;
1144 
1145         xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);
1146 
1147         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1148          * part of the resource name, so that bundles don't restart in a rolling
1149          * upgrade. (It also avoids needing to change regression tests.)
1150          */
1151         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1152                       (bundle_data->promoted_max? "master"
1153                       : (const char *)xml_resource->name));
1154 
1155         xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
1156         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1157 
1158         crm_create_nvpair_xml(xml_set, NULL,
1159                               PCMK_META_ORDERED, PCMK_VALUE_TRUE);
1160 
1161         value = pcmk__itoa(bundle_data->nreplicas);
1162         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
1163         free(value);
1164 
1165         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1166         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
1167         free(value);
1168 
1169         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
1170                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1171 
1172         if (bundle_data->promoted_max) {
1173             crm_create_nvpair_xml(xml_set, NULL,
1174                                   PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);
1175 
1176             value = pcmk__itoa(bundle_data->promoted_max);
1177             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
1178             free(value);
1179         }
1180 
1181         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
1182         pcmk__xml_copy(xml_resource, xml_obj);
1183 
1184     } else if(xml_obj) {
1185         pcmk__config_err("Cannot control %s inside %s without either "
1186                          PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
1187                          rsc->id, pcmk__xe_id(xml_obj));
1188         return FALSE;
1189     }
1190 
1191     if(xml_resource) {
1192         int lpc = 0;
1193         GList *childIter = NULL;
1194         pe__bundle_port_t *port = NULL;
1195         GString *buffer = NULL;
1196 
1197         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
1198                                 scheduler) != pcmk_rc_ok) {
1199             return FALSE;
1200         }
1201 
1202         /* Currently, we always map the default authentication key location
1203          * into the same location inside the container.
1204          *
1205          * Ideally, we would respect the host's PCMK_authkey_location, but:
1206          * - it may be different on different nodes;
1207          * - the actual connection will do extra checking to make sure the key
1208          *   file exists and is readable, that we can't do here on the DC
1209          * - tools such as crm_resource and crm_simulate may not have the same
1210          *   environment variables as the cluster, causing operation digests to
1211          *   differ
1212          *
1213          * Always using the default location inside the container is fine,
1214          * because we control the pacemaker_remote environment, and it avoids
1215          * having to pass another environment variable to the container.
1216          *
1217          * @TODO A better solution may be to have only pacemaker_remote use the
1218          * environment variable, and have the cluster nodes use a new
1219          * cluster option for key location. This would introduce the limitation
1220          * of the location being the same on all cluster nodes, but that's
1221          * reasonable.
1222          */
1223         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1224                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1225 
1226         if (need_log_mount) {
1227             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1228                       pe__bundle_mount_subdir);
1229         }
1230 
1231         port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1232         if(bundle_data->control_port) {
1233             port->source = strdup(bundle_data->control_port);
1234         } else {
1235             /* If we wanted to respect PCMK_remote_port, we could use
1236              * crm_default_remote_port() here and elsewhere in this file instead
1237              * of DEFAULT_REMOTE_PORT.
1238              *
1239              * However, it gains nothing, since we control both the container
1240              * environment and the connection resource parameters, and the user
1241              * can use a different port if desired by setting
1242              * PCMK_XA_CONTROL_PORT.
1243              */
1244             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1245         }
1246         port->target = strdup(port->source);
1247         bundle_data->ports = g_list_append(bundle_data->ports, port);
1248 
1249         buffer = g_string_sized_new(1024);
1250         for (childIter = bundle_data->child->children; childIter != NULL;
1251              childIter = childIter->next) {
1252 
1253             pcmk__bundle_replica_t *replica = NULL;
1254 
1255             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1256             replica->child = childIter->data;
1257             replica->child->exclusive_discover = TRUE;
1258             replica->offset = lpc++;
1259 
1260             // Ensure the child's notify gets set based on the underlying primitive's value
1261             if (pcmk_is_set(replica->child->flags, pcmk_rsc_notify)) {
1262                 pcmk__set_rsc_flags(bundle_data->child, pcmk_rsc_notify);
1263             }
1264 
1265             allocate_ip(bundle_data, replica, buffer);
1266             bundle_data->replicas = g_list_append(bundle_data->replicas,
1267                                                   replica);
1268             bundle_data->attribute_target =
1269                 g_hash_table_lookup(replica->child->meta,
1270                                     PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
1271         }
1272         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1273 
1274         if (bundle_data->attribute_target) {
1275             pcmk__insert_dup(rsc->meta, PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1276                              bundle_data->attribute_target);
1277             pcmk__insert_dup(bundle_data->child->meta,
1278                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1279                              bundle_data->attribute_target);
1280         }
1281 
1282     } else {
1283         // Just a naked container, no pacemaker-remote
1284         GString *buffer = g_string_sized_new(1024);
1285 
1286         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1287             pcmk__bundle_replica_t *replica = NULL;
1288 
1289             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1290             replica->offset = lpc;
1291             allocate_ip(bundle_data, replica, buffer);
1292             bundle_data->replicas = g_list_append(bundle_data->replicas,
1293                                                   replica);
1294         }
1295         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1296     }
1297 
1298     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1299          gIter = gIter->next) {
1300         pcmk__bundle_replica_t *replica = gIter->data;
1301 
1302         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1303             pcmk__config_err("Failed unpacking resource %s", rsc->id);
1304             rsc->fns->free(rsc);
1305             return FALSE;
1306         }
1307 
1308         /* Utilization needs special handling for bundles. It makes no sense for
1309          * the inner primitive to have utilization, because it is tied
1310          * one-to-one to the guest node created by the container resource -- and
1311          * there's no way to set capacities for that guest node anyway.
1312          *
1313          * What the user really wants is to configure utilization for the
1314          * container. However, the schema only allows utilization for
1315          * primitives, and the container resource is implicit anyway, so the
1316          * user can *only* configure utilization for the inner primitive. If
1317          * they do, move the primitive's utilization values to the container.
1318          *
1319          * @TODO This means that bundles without an inner primitive can't have
1320          * utilization. An alternative might be to allow utilization values in
1321          * the top-level bundle XML in the schema, and copy those to each
1322          * container.
1323          */
1324         if (replica->child != NULL) {
1325             GHashTable *empty = replica->container->utilization;
1326 
1327             replica->container->utilization = replica->child->utilization;
1328             replica->child->utilization = empty;
1329         }
1330     }
1331 
1332     if (bundle_data->child) {
1333         rsc->children = g_list_append(rsc->children, bundle_data->child);
1334     }
1335     return TRUE;
1336 }
1337 
1338 static int
1339 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1340 {
1341     if (rsc) {
1342         gboolean child_active = rsc->fns->active(rsc, all);
1343 
1344         if (child_active && !all) {
1345             return TRUE;
1346         } else if (!child_active && all) {
1347             return FALSE;
1348         }
1349     }
1350     return -1;
1351 }
1352 
1353 gboolean
1354 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1355 {
1356     pe__bundle_variant_data_t *bundle_data = NULL;
1357     GList *iter = NULL;
1358 
1359     get_bundle_variant_data(bundle_data, rsc);
1360     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1361         pcmk__bundle_replica_t *replica = iter->data;
1362         int rsc_active;
1363 
1364         rsc_active = replica_resource_active(replica->ip, all);
1365         if (rsc_active >= 0) {
1366             return (gboolean) rsc_active;
1367         }
1368 
1369         rsc_active = replica_resource_active(replica->child, all);
1370         if (rsc_active >= 0) {
1371             return (gboolean) rsc_active;
1372         }
1373 
1374         rsc_active = replica_resource_active(replica->container, all);
1375         if (rsc_active >= 0) {
1376             return (gboolean) rsc_active;
1377         }
1378 
1379         rsc_active = replica_resource_active(replica->remote, all);
1380         if (rsc_active >= 0) {
1381             return (gboolean) rsc_active;
1382         }
1383     }
1384 
1385     /* If "all" is TRUE, we've already checked that no resources were inactive,
1386      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1387      * so return FALSE.
1388      */
1389     return all;
1390 }
1391 
1392 /*!
1393  * \internal
1394  * \brief Find the bundle replica corresponding to a given node
1395  *
1396  * \param[in] bundle  Top-level bundle resource
1397  * \param[in] node    Node to search for
1398  *
1399  * \return Bundle replica if found, NULL otherwise
1400  */
1401 pcmk_resource_t *
1402 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
     /*  */
1403 {
1404     pe__bundle_variant_data_t *bundle_data = NULL;
1405     CRM_ASSERT(bundle && node);
1406 
1407     get_bundle_variant_data(bundle_data, bundle);
1408     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1409          gIter = gIter->next) {
1410         pcmk__bundle_replica_t *replica = gIter->data;
1411 
1412         CRM_ASSERT(replica && replica->node);
1413         if (pcmk__same_node(replica->node, node)) {
1414             return replica->child;
1415         }
1416     }
1417     return NULL;
1418 }
1419 
1420 /*!
1421  * \internal
1422  * \deprecated This function will be removed in a future release
1423  */
1424 static void
1425 print_rsc_in_list(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1426                   void *print_data)
1427 {
1428     if (rsc != NULL) {
1429         if (options & pe_print_html) {
1430             status_print("<li>");
1431         }
1432         rsc->fns->print(rsc, pre_text, options, print_data);
1433         if (options & pe_print_html) {
1434             status_print("</li>\n");
1435         }
1436     }
1437 }
1438 
1439 /*!
1440  * \internal
1441  * \deprecated This function will be removed in a future release
1442  */
1443 static void
1444 bundle_print_xml(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1445                  void *print_data)
1446 {
1447     pe__bundle_variant_data_t *bundle_data = NULL;
1448     char *child_text = NULL;
1449     CRM_CHECK(rsc != NULL, return);
1450 
1451     if (pre_text == NULL) {
1452         pre_text = "";
1453     }
1454     child_text = crm_strdup_printf("%s        ", pre_text);
1455 
1456     get_bundle_variant_data(bundle_data, rsc);
1457 
1458     status_print("%s<bundle ", pre_text);
1459     status_print(PCMK_XA_ID "=\"%s\" ", rsc->id);
1460     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1461     status_print("image=\"%s\" ", bundle_data->image);
1462     status_print("unique=\"%s\" ",
1463                  pcmk__flag_text(rsc->flags, pcmk_rsc_unique));
1464     status_print("managed=\"%s\" ",
1465                  pcmk__flag_text(rsc->flags, pcmk_rsc_managed));
1466     status_print("failed=\"%s\" ",
1467                  pcmk__flag_text(rsc->flags, pcmk_rsc_failed));
1468     status_print(">\n");
1469 
1470     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1471          gIter = gIter->next) {
1472         pcmk__bundle_replica_t *replica = gIter->data;
1473 
1474         CRM_ASSERT(replica);
1475         status_print("%s    <replica " PCMK_XA_ID "=\"%d\">\n",
1476                      pre_text, replica->offset);
1477         print_rsc_in_list(replica->ip, child_text, options, print_data);
1478         print_rsc_in_list(replica->child, child_text, options, print_data);
1479         print_rsc_in_list(replica->container, child_text, options, print_data);
1480         print_rsc_in_list(replica->remote, child_text, options, print_data);
1481         status_print("%s    </replica>\n", pre_text);
1482     }
1483     status_print("%s</bundle>\n", pre_text);
1484     free(child_text);
1485 }
1486 
1487 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1488                   "GList *")
1489 int
1490 pe__bundle_xml(pcmk__output_t *out, va_list args)
1491 {
1492     uint32_t show_opts = va_arg(args, uint32_t);
1493     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1494     GList *only_node = va_arg(args, GList *);
1495     GList *only_rsc = va_arg(args, GList *);
1496 
1497     pe__bundle_variant_data_t *bundle_data = NULL;
1498     int rc = pcmk_rc_no_output;
1499     gboolean printed_header = FALSE;
1500     gboolean print_everything = TRUE;
1501 
1502     const char *desc = NULL;
1503 
1504     CRM_ASSERT(rsc != NULL);
1505     
1506     get_bundle_variant_data(bundle_data, rsc);
1507 
1508     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1509         return rc;
1510     }
1511 
1512     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1513 
1514     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1515          gIter = gIter->next) {
1516         pcmk__bundle_replica_t *replica = gIter->data;
1517         char *id = NULL;
1518         gboolean print_ip, print_child, print_ctnr, print_remote;
1519 
1520         CRM_ASSERT(replica);
1521 
1522         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1523             continue;
1524         }
1525 
1526         print_ip = replica->ip != NULL &&
1527                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1528         print_child = replica->child != NULL &&
1529                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1530         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1531         print_remote = replica->remote != NULL &&
1532                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1533 
1534         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1535             continue;
1536         }
1537 
1538         if (!printed_header) {
1539             const char *type = container_agent_str(bundle_data->agent_type);
1540             const char *unique = pcmk__flag_text(rsc->flags, pcmk_rsc_unique);
1541             const char *maintenance = pcmk__flag_text(rsc->flags,
1542                                                       pcmk_rsc_maintenance);
1543             const char *managed = pcmk__flag_text(rsc->flags, pcmk_rsc_managed);
1544             const char *failed = pcmk__flag_text(rsc->flags, pcmk_rsc_failed);
1545 
1546             printed_header = TRUE;
1547 
1548             desc = pe__resource_description(rsc, show_opts);
1549 
1550             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
1551                                           PCMK_XA_ID, rsc->id,
1552                                           PCMK_XA_TYPE, type,
1553                                           PCMK_XA_IMAGE, bundle_data->image,
1554                                           PCMK_XA_UNIQUE, unique,
1555                                           PCMK_XA_MAINTENANCE, maintenance,
1556                                           PCMK_XA_MANAGED, managed,
1557                                           PCMK_XA_FAILED, failed,
1558                                           PCMK_XA_DESCRIPTION, desc,
1559                                           NULL);
1560             CRM_ASSERT(rc == pcmk_rc_ok);
1561         }
1562 
1563         id = pcmk__itoa(replica->offset);
1564         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
1565                                       PCMK_XA_ID, id,
1566                                       NULL);
1567         free(id);
1568         CRM_ASSERT(rc == pcmk_rc_ok);
1569 
1570         if (print_ip) {
1571             out->message(out, (const char *) replica->ip->xml->name, show_opts,
1572                          replica->ip, only_node, only_rsc);
1573         }
1574 
1575         if (print_child) {
1576             out->message(out, (const char *) replica->child->xml->name,
1577                          show_opts, replica->child, only_node, only_rsc);
1578         }
1579 
1580         if (print_ctnr) {
1581             out->message(out, (const char *) replica->container->xml->name,
1582                          show_opts, replica->container, only_node, only_rsc);
1583         }
1584 
1585         if (print_remote) {
1586             out->message(out, (const char *) replica->remote->xml->name,
1587                          show_opts, replica->remote, only_node, only_rsc);
1588         }
1589 
1590         pcmk__output_xml_pop_parent(out); // replica
1591     }
1592 
1593     if (printed_header) {
1594         pcmk__output_xml_pop_parent(out); // bundle
1595     }
1596 
1597     return rc;
1598 }
1599 
1600 static void
1601 pe__bundle_replica_output_html(pcmk__output_t *out,
     /*  */
1602                                pcmk__bundle_replica_t *replica,
1603                                pcmk_node_t *node, uint32_t show_opts)
1604 {
1605     pcmk_resource_t *rsc = replica->child;
1606 
1607     int offset = 0;
1608     char buffer[LINE_MAX];
1609 
1610     if(rsc == NULL) {
1611         rsc = replica->container;
1612     }
1613 
1614     if (replica->remote) {
1615         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1616                            rsc_printable_id(replica->remote));
1617     } else {
1618         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1619                            rsc_printable_id(replica->container));
1620     }
1621     if (replica->ipaddr) {
1622         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1623                            replica->ipaddr);
1624     }
1625 
1626     pe__common_output_html(out, rsc, buffer, node, show_opts);
1627 }
1628 
1629 /*!
1630  * \internal
1631  * \brief Get a string describing a resource's unmanaged state or lack thereof
1632  *
1633  * \param[in] rsc  Resource to describe
1634  *
1635  * \return A string indicating that a resource is in maintenance mode or
1636  *         otherwise unmanaged, or an empty string otherwise
1637  */
1638 static const char *
1639 get_unmanaged_str(const pcmk_resource_t *rsc)
     /*  */
1640 {
1641     if (pcmk_is_set(rsc->flags, pcmk_rsc_maintenance)) {
1642         return " (maintenance)";
1643     }
1644     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
1645         return " (unmanaged)";
1646     }
1647     return "";
1648 }
1649 
1650 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1651                   "GList *")
1652 int
1653 pe__bundle_html(pcmk__output_t *out, va_list args)
1654 {
1655     uint32_t show_opts = va_arg(args, uint32_t);
1656     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1657     GList *only_node = va_arg(args, GList *);
1658     GList *only_rsc = va_arg(args, GList *);
1659 
1660     const char *desc = NULL;
1661     pe__bundle_variant_data_t *bundle_data = NULL;
1662     int rc = pcmk_rc_no_output;
1663     gboolean print_everything = TRUE;
1664 
1665     CRM_ASSERT(rsc != NULL);
1666 
1667     get_bundle_variant_data(bundle_data, rsc);
1668 
1669     desc = pe__resource_description(rsc, show_opts);
1670 
1671     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1672         return rc;
1673     }
1674 
1675     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1676 
1677     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1678          gIter = gIter->next) {
1679         pcmk__bundle_replica_t *replica = gIter->data;
1680         gboolean print_ip, print_child, print_ctnr, print_remote;
1681 
1682         CRM_ASSERT(replica);
1683 
1684         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1685             continue;
1686         }
1687 
1688         print_ip = replica->ip != NULL &&
1689                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1690         print_child = replica->child != NULL &&
1691                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1692         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1693         print_remote = replica->remote != NULL &&
1694                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1695 
1696         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1697             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1698             /* The text output messages used below require pe_print_implicit to
1699              * be set to do anything.
1700              */
1701             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1702 
1703             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1704                                      (bundle_data->nreplicas > 1)? " set" : "",
1705                                      rsc->id, bundle_data->image,
1706                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1707                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1708                                      get_unmanaged_str(rsc));
1709 
1710             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1711                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1712             }
1713 
1714             if (print_ip) {
1715                 out->message(out, (const char *) replica->ip->xml->name,
1716                              new_show_opts, replica->ip, only_node, only_rsc);
1717             }
1718 
1719             if (print_child) {
1720                 out->message(out, (const char *) replica->child->xml->name,
1721                              new_show_opts, replica->child, only_node,
1722                              only_rsc);
1723             }
1724 
1725             if (print_ctnr) {
1726                 out->message(out, (const char *) replica->container->xml->name,
1727                              new_show_opts, replica->container, only_node,
1728                              only_rsc);
1729             }
1730 
1731             if (print_remote) {
1732                 out->message(out, (const char *) replica->remote->xml->name,
1733                              new_show_opts, replica->remote, only_node,
1734                              only_rsc);
1735             }
1736 
1737             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1738                 out->end_list(out);
1739             }
1740         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1741             continue;
1742         } else {
1743             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1744                                      (bundle_data->nreplicas > 1)? " set" : "",
1745                                      rsc->id, bundle_data->image,
1746                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1747                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1748                                      get_unmanaged_str(rsc));
1749 
1750             pe__bundle_replica_output_html(out, replica,
1751                                            pcmk__current_node(replica->container),
1752                                            show_opts);
1753         }
1754     }
1755 
1756     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1757     return rc;
1758 }
1759 
1760 static void
1761 pe__bundle_replica_output_text(pcmk__output_t *out,
     /*  */
1762                                pcmk__bundle_replica_t *replica,
1763                                pcmk_node_t *node, uint32_t show_opts)
1764 {
1765     const pcmk_resource_t *rsc = replica->child;
1766 
1767     int offset = 0;
1768     char buffer[LINE_MAX];
1769 
1770     if(rsc == NULL) {
1771         rsc = replica->container;
1772     }
1773 
1774     if (replica->remote) {
1775         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1776                            rsc_printable_id(replica->remote));
1777     } else {
1778         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1779                            rsc_printable_id(replica->container));
1780     }
1781     if (replica->ipaddr) {
1782         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1783                            replica->ipaddr);
1784     }
1785 
1786     pe__common_output_text(out, rsc, buffer, node, show_opts);
1787 }
1788 
1789 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1790                   "GList *")
1791 int
1792 pe__bundle_text(pcmk__output_t *out, va_list args)
1793 {
1794     uint32_t show_opts = va_arg(args, uint32_t);
1795     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1796     GList *only_node = va_arg(args, GList *);
1797     GList *only_rsc = va_arg(args, GList *);
1798 
1799     const char *desc = NULL;
1800     pe__bundle_variant_data_t *bundle_data = NULL;
1801     int rc = pcmk_rc_no_output;
1802     gboolean print_everything = TRUE;
1803 
1804     desc = pe__resource_description(rsc, show_opts);
1805     
1806     get_bundle_variant_data(bundle_data, rsc);
1807 
1808     CRM_ASSERT(rsc != NULL);
1809 
1810     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1811         return rc;
1812     }
1813 
1814     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1815 
1816     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1817          gIter = gIter->next) {
1818         pcmk__bundle_replica_t *replica = gIter->data;
1819         gboolean print_ip, print_child, print_ctnr, print_remote;
1820 
1821         CRM_ASSERT(replica);
1822 
1823         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1824             continue;
1825         }
1826 
1827         print_ip = replica->ip != NULL &&
1828                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1829         print_child = replica->child != NULL &&
1830                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1831         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1832         print_remote = replica->remote != NULL &&
1833                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1834 
1835         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1836             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1837             /* The text output messages used below require pe_print_implicit to
1838              * be set to do anything.
1839              */
1840             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1841 
1842             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1843                                      (bundle_data->nreplicas > 1)? " set" : "",
1844                                      rsc->id, bundle_data->image,
1845                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1846                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1847                                      get_unmanaged_str(rsc));
1848 
1849             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1850                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1851             }
1852 
1853             out->begin_list(out, NULL, NULL, NULL);
1854 
1855             if (print_ip) {
1856                 out->message(out, (const char *) replica->ip->xml->name,
1857                              new_show_opts, replica->ip, only_node, only_rsc);
1858             }
1859 
1860             if (print_child) {
1861                 out->message(out, (const char *) replica->child->xml->name,
1862                              new_show_opts, replica->child, only_node,
1863                              only_rsc);
1864             }
1865 
1866             if (print_ctnr) {
1867                 out->message(out, (const char *) replica->container->xml->name,
1868                              new_show_opts, replica->container, only_node,
1869                              only_rsc);
1870             }
1871 
1872             if (print_remote) {
1873                 out->message(out, (const char *) replica->remote->xml->name,
1874                              new_show_opts, replica->remote, only_node,
1875                              only_rsc);
1876             }
1877 
1878             out->end_list(out);
1879         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1880             continue;
1881         } else {
1882             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1883                                      (bundle_data->nreplicas > 1)? " set" : "",
1884                                      rsc->id, bundle_data->image,
1885                                      pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1886                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1887                                      get_unmanaged_str(rsc));
1888 
1889             pe__bundle_replica_output_text(out, replica,
1890                                            pcmk__current_node(replica->container),
1891                                            show_opts);
1892         }
1893     }
1894 
1895     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1896     return rc;
1897 }
1898 
1899 /*!
1900  * \internal
1901  * \deprecated This function will be removed in a future release
1902  */
1903 static void
1904 print_bundle_replica(pcmk__bundle_replica_t *replica, const char *pre_text,
     /*  */
1905                      long options, void *print_data)
1906 {
1907     pcmk_node_t *node = NULL;
1908     pcmk_resource_t *rsc = replica->child;
1909 
1910     int offset = 0;
1911     char buffer[LINE_MAX];
1912 
1913     if(rsc == NULL) {
1914         rsc = replica->container;
1915     }
1916 
1917     if (replica->remote) {
1918         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1919                            rsc_printable_id(replica->remote));
1920     } else {
1921         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1922                            rsc_printable_id(replica->container));
1923     }
1924     if (replica->ipaddr) {
1925         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1926                            replica->ipaddr);
1927     }
1928 
1929     node = pcmk__current_node(replica->container);
1930     common_print(rsc, pre_text, buffer, node, options, print_data);
1931 }
1932 
1933 /*!
1934  * \internal
1935  * \deprecated This function will be removed in a future release
1936  */
1937 void
1938 pe__print_bundle(pcmk_resource_t *rsc, const char *pre_text, long options,
     /*  */
1939                  void *print_data)
1940 {
1941     pe__bundle_variant_data_t *bundle_data = NULL;
1942     char *child_text = NULL;
1943     CRM_CHECK(rsc != NULL, return);
1944 
1945     if (options & pe_print_xml) {
1946         bundle_print_xml(rsc, pre_text, options, print_data);
1947         return;
1948     }
1949 
1950     get_bundle_variant_data(bundle_data, rsc);
1951 
1952     if (pre_text == NULL) {
1953         pre_text = " ";
1954     }
1955 
1956     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1957                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1958                  rsc->id, bundle_data->image,
1959                  pcmk_is_set(rsc->flags, pcmk_rsc_unique)? " (unique)" : "",
1960                  pcmk_is_set(rsc->flags, pcmk_rsc_managed)? "" : " (unmanaged)");
1961     if (options & pe_print_html) {
1962         status_print("<br />\n<ul>\n");
1963     }
1964 
1965 
1966     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1967          gIter = gIter->next) {
1968         pcmk__bundle_replica_t *replica = gIter->data;
1969 
1970         CRM_ASSERT(replica);
1971         if (options & pe_print_html) {
1972             status_print("<li>");
1973         }
1974 
1975         if (pcmk_is_set(options, pe_print_implicit)) {
1976             child_text = crm_strdup_printf("     %s", pre_text);
1977             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1978                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1979             }
1980             if (options & pe_print_html) {
1981                 status_print("<br />\n<ul>\n");
1982             }
1983             print_rsc_in_list(replica->ip, child_text, options, print_data);
1984             print_rsc_in_list(replica->container, child_text, options, print_data);
1985             print_rsc_in_list(replica->remote, child_text, options, print_data);
1986             print_rsc_in_list(replica->child, child_text, options, print_data);
1987             if (options & pe_print_html) {
1988                 status_print("</ul>\n");
1989             }
1990         } else {
1991             child_text = crm_strdup_printf("%s  ", pre_text);
1992             print_bundle_replica(replica, child_text, options, print_data);
1993         }
1994         free(child_text);
1995 
1996         if (options & pe_print_html) {
1997             status_print("</li>\n");
1998         }
1999     }
2000     if (options & pe_print_html) {
2001         status_print("</ul>\n");
2002     }
2003 }
2004 
2005 static void
2006 free_bundle_replica(pcmk__bundle_replica_t *replica)
     /*  */
2007 {
2008     if (replica == NULL) {
2009         return;
2010     }
2011 
2012     if (replica->node) {
2013         free(replica->node);
2014         replica->node = NULL;
2015     }
2016 
2017     if (replica->ip) {
2018         free_xml(replica->ip->xml);
2019         replica->ip->xml = NULL;
2020         replica->ip->fns->free(replica->ip);
2021         replica->ip = NULL;
2022     }
2023     if (replica->container) {
2024         free_xml(replica->container->xml);
2025         replica->container->xml = NULL;
2026         replica->container->fns->free(replica->container);
2027         replica->container = NULL;
2028     }
2029     if (replica->remote) {
2030         free_xml(replica->remote->xml);
2031         replica->remote->xml = NULL;
2032         replica->remote->fns->free(replica->remote);
2033         replica->remote = NULL;
2034     }
2035     free(replica->ipaddr);
2036     free(replica);
2037 }
2038 
2039 void
2040 pe__free_bundle(pcmk_resource_t *rsc)
     /*  */
2041 {
2042     pe__bundle_variant_data_t *bundle_data = NULL;
2043     CRM_CHECK(rsc != NULL, return);
2044 
2045     get_bundle_variant_data(bundle_data, rsc);
2046     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
2047 
2048     free(bundle_data->prefix);
2049     free(bundle_data->image);
2050     free(bundle_data->control_port);
2051     free(bundle_data->host_network);
2052     free(bundle_data->host_netmask);
2053     free(bundle_data->ip_range_start);
2054     free(bundle_data->container_network);
2055     free(bundle_data->launcher_options);
2056     free(bundle_data->container_command);
2057     g_free(bundle_data->container_host_options);
2058 
2059     g_list_free_full(bundle_data->replicas,
2060                      (GDestroyNotify) free_bundle_replica);
2061     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
2062     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
2063     g_list_free(rsc->children);
2064 
2065     if(bundle_data->child) {
2066         free_xml(bundle_data->child->xml);
2067         bundle_data->child->xml = NULL;
2068         bundle_data->child->fns->free(bundle_data->child);
2069     }
2070     common_free(rsc);
2071 }
2072 
2073 enum rsc_role_e
2074 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
     /*  */
2075 {
2076     enum rsc_role_e container_role = pcmk_role_unknown;
2077     return container_role;
2078 }
2079 
2080 /*!
2081  * \brief Get the number of configured replicas in a bundle
2082  *
2083  * \param[in] rsc  Bundle resource
2084  *
2085  * \return Number of configured replicas, or 0 on error
2086  */
2087 int
2088 pe_bundle_replicas(const pcmk_resource_t *rsc)
     /*  */
2089 {
2090     if (pcmk__is_bundle(rsc)) {
2091         pe__bundle_variant_data_t *bundle_data = NULL;
2092 
2093         get_bundle_variant_data(bundle_data, rsc);
2094         return bundle_data->nreplicas;
2095     }
2096     return 0;
2097 }
2098 
2099 void
2100 pe__count_bundle(pcmk_resource_t *rsc)
     /*  */
2101 {
2102     pe__bundle_variant_data_t *bundle_data = NULL;
2103 
2104     get_bundle_variant_data(bundle_data, rsc);
2105     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2106         pcmk__bundle_replica_t *replica = item->data;
2107 
2108         if (replica->ip) {
2109             replica->ip->fns->count(replica->ip);
2110         }
2111         if (replica->child) {
2112             replica->child->fns->count(replica->child);
2113         }
2114         if (replica->container) {
2115             replica->container->fns->count(replica->container);
2116         }
2117         if (replica->remote) {
2118             replica->remote->fns->count(replica->remote);
2119         }
2120     }
2121 }
2122 
2123 gboolean
2124 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
     /*  */
2125                        gboolean check_parent)
2126 {
2127     gboolean passes = FALSE;
2128     pe__bundle_variant_data_t *bundle_data = NULL;
2129 
2130     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
2131         passes = TRUE;
2132     } else {
2133         get_bundle_variant_data(bundle_data, rsc);
2134 
2135         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2136             pcmk__bundle_replica_t *replica = gIter->data;
2137 
2138             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2139                 passes = TRUE;
2140                 break;
2141             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2142                 passes = TRUE;
2143                 break;
2144             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2145                 passes = TRUE;
2146                 break;
2147             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2148                 passes = TRUE;
2149                 break;
2150             }
2151         }
2152     }
2153 
2154     return !passes;
2155 }
2156 
2157 /*!
2158  * \internal
2159  * \brief Get a list of a bundle's containers
2160  *
2161  * \param[in] bundle  Bundle resource
2162  *
2163  * \return Newly created list of \p bundle's containers
2164  * \note It is the caller's responsibility to free the result with
2165  *       g_list_free().
2166  */
2167 GList *
2168 pe__bundle_containers(const pcmk_resource_t *bundle)
     /*  */
2169 {
2170     GList *containers = NULL;
2171     const pe__bundle_variant_data_t *data = NULL;
2172 
2173     get_bundle_variant_data(data, bundle);
2174     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
2175         pcmk__bundle_replica_t *replica = iter->data;
2176 
2177         containers = g_list_append(containers, replica->container);
2178     }
2179     return containers;
2180 }
2181 
2182 // Bundle implementation of pcmk_rsc_methods_t:active_node()
2183 pcmk_node_t *
2184 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
     /*  */
2185                        unsigned int *count_clean)
2186 {
2187     pcmk_node_t *active = NULL;
2188     pcmk_node_t *node = NULL;
2189     pcmk_resource_t *container = NULL;
2190     GList *containers = NULL;
2191     GList *iter = NULL;
2192     GHashTable *nodes = NULL;
2193     const pe__bundle_variant_data_t *data = NULL;
2194 
2195     if (count_all != NULL) {
2196         *count_all = 0;
2197     }
2198     if (count_clean != NULL) {
2199         *count_clean = 0;
2200     }
2201     if (rsc == NULL) {
2202         return NULL;
2203     }
2204 
2205     /* For the purposes of this method, we only care about where the bundle's
2206      * containers are active, so build a list of active containers.
2207      */
2208     get_bundle_variant_data(data, rsc);
2209     for (iter = data->replicas; iter != NULL; iter = iter->next) {
2210         pcmk__bundle_replica_t *replica = iter->data;
2211 
2212         if (replica->container->running_on != NULL) {
2213             containers = g_list_append(containers, replica->container);
2214         }
2215     }
2216     if (containers == NULL) {
2217         return NULL;
2218     }
2219 
2220     /* If the bundle has only a single active container, just use that
2221      * container's method. If live migration is ever supported for bundle
2222      * containers, this will allow us to prefer the migration source when there
2223      * is only one container and it is migrating. For now, this just lets us
2224      * avoid creating the nodes table.
2225      */
2226     if (pcmk__list_of_1(containers)) {
2227         container = containers->data;
2228         node = container->fns->active_node(container, count_all, count_clean);
2229         g_list_free(containers);
2230         return node;
2231     }
2232 
2233     // Add all containers' active nodes to a hash table (for uniqueness)
2234     nodes = g_hash_table_new(NULL, NULL);
2235     for (iter = containers; iter != NULL; iter = iter->next) {
2236         container = iter->data;
2237 
2238         for (GList *node_iter = container->running_on; node_iter != NULL;
2239              node_iter = node_iter->next) {
2240             node = node_iter->data;
2241 
2242             // If insert returns true, we haven't counted this node yet
2243             if (g_hash_table_insert(nodes, (gpointer) node->details,
2244                                     (gpointer) node)
2245                 && !pe__count_active_node(rsc, node, &active, count_all,
2246                                           count_clean)) {
2247                 goto done;
2248             }
2249         }
2250     }
2251 
2252 done:
2253     g_list_free(containers);
2254     g_hash_table_destroy(nodes);
2255     return active;
2256 }
2257 
2258 /*!
2259  * \internal
2260  * \brief Get maximum bundle resource instances per node
2261  *
2262  * \param[in] rsc  Bundle resource to check
2263  *
2264  * \return Maximum number of \p rsc instances that can be active on one node
2265  */
2266 unsigned int
2267 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
     /*  */
2268 {
2269     pe__bundle_variant_data_t *bundle_data = NULL;
2270 
2271     get_bundle_variant_data(bundle_data, rsc);
2272     CRM_ASSERT(bundle_data->nreplicas_per_host >= 0);
2273     return (unsigned int) bundle_data->nreplicas_per_host;
2274 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
