lib/pengine/bundle.c

/* */
This source file includes following definitions.
pe__bundle_max
pe__bundled_resource
pe__get_rsc_in_container
pe__node_is_bundle_instance
pe__first_container
pe__foreach_bundle_replica
pe__foreach_const_bundle_replica
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
container_agent_str
create_container_resource
disallow_node
create_remote_resource
create_replica_resources
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
get_unmanaged_str
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
pe__bundle_containers
pe__bundle_active_node
pe__bundle_max_per_node
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/common/xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 enum pe__bundle_mount_flags {
  24     pe__bundle_mount_none       = 0x00,
  25 
  26     // mount instance-specific subdirectory rather than source directly
  27     pe__bundle_mount_subdir     = 0x01
  28 };
  29 
  30 typedef struct {
  31     char *source;
  32     char *target;
  33     char *options;
  34     uint32_t flags; // bitmask of pe__bundle_mount_flags
  35 } pe__bundle_mount_t;
  36 
  37 typedef struct {
  38     char *source;
  39     char *target;
  40 } pe__bundle_port_t;
  41 
  42 enum pe__container_agent {
  43     PE__CONTAINER_AGENT_UNKNOWN,
  44     PE__CONTAINER_AGENT_DOCKER,
  45     PE__CONTAINER_AGENT_PODMAN,
  46 };
  47 
  48 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
  49 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
  50 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
  51 
  52 typedef struct pe__bundle_variant_data_s {
  53         int promoted_max;
  54         int nreplicas;
  55         int nreplicas_per_host;
  56         char *prefix;
  57         char *image;
  58         const char *ip_last;
  59         char *host_network;
  60         char *host_netmask;
  61         char *control_port;
  62         char *container_network;
  63         char *ip_range_start;
  64         gboolean add_host;
  65         gchar *container_host_options;
  66         char *container_command;
  67         char *launcher_options;
  68         const char *attribute_target;
  69 
  70         pcmk_resource_t *child;
  71 
  72         GList *replicas;    // pcmk__bundle_replica_t *
  73         GList *ports;       // pe__bundle_port_t *
  74         GList *mounts;      // pe__bundle_mount_t *
  75 
  76         enum pe__container_agent agent_type;
  77 } pe__bundle_variant_data_t;
  78 
  79 #define get_bundle_variant_data(data, rsc) do { \
  80         pcmk__assert(pcmk__is_bundle(rsc));     \
  81         data = rsc->priv->variant_opaque;       \
  82     } while (0)
  83 
  84 /*!
  85  * \internal
  86  * \brief Get maximum number of bundle replicas allowed to run
  87  *
  88  * \param[in] rsc  Bundle or bundled resource to check
  89  *
  90  * \return Maximum replicas for bundle corresponding to \p rsc
  91  */
  92 int
  93 pe__bundle_max(const pcmk_resource_t *rsc)
     /*  */
  94 {
  95     const pe__bundle_variant_data_t *bundle_data = NULL;
  96 
  97     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
  98     return bundle_data->nreplicas;
  99 }
 100 
 101 /*!
 102  * \internal
 103  * \brief Get the resource inside a bundle
 104  *
 105  * \param[in] bundle  Bundle to check
 106  *
 107  * \return Resource inside \p bundle if any, otherwise NULL
 108  */
 109 pcmk_resource_t *
 110 pe__bundled_resource(const pcmk_resource_t *rsc)
     /*  */
 111 {
 112     const pe__bundle_variant_data_t *bundle_data = NULL;
 113 
 114     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 115     return bundle_data->child;
 116 }
 117 
 118 /*!
 119  * \internal
 120  * \brief Get containerized resource corresponding to a given bundle container
 121  *
 122  * \param[in] instance  Collective instance that might be a bundle container
 123  *
 124  * \return Bundled resource instance inside \p instance if it is a bundle
 125  *         container instance, otherwise NULL
 126  */
 127 const pcmk_resource_t *
 128 pe__get_rsc_in_container(const pcmk_resource_t *instance)
     /*  */
 129 {
 130     const pe__bundle_variant_data_t *data = NULL;
 131     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 132 
 133     if (!pcmk__is_bundle(top)) {
 134         return NULL;
 135     }
 136     get_bundle_variant_data(data, top);
 137 
 138     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 139         const pcmk__bundle_replica_t *replica = iter->data;
 140 
 141         if (instance == replica->container) {
 142             return replica->child;
 143         }
 144     }
 145     return NULL;
 146 }
 147 
 148 /*!
 149  * \internal
 150  * \brief Check whether a given node is created by a bundle
 151  *
 152  * \param[in] bundle  Bundle resource to check
 153  * \param[in] node    Node to check
 154  *
 155  * \return true if \p node is an instance of \p bundle, otherwise false
 156  */
 157 bool
 158 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
     /*  */
 159                             const pcmk_node_t *node)
 160 {
 161     pe__bundle_variant_data_t *bundle_data = NULL;
 162 
 163     get_bundle_variant_data(bundle_data, bundle);
 164     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 165         pcmk__bundle_replica_t *replica = iter->data;
 166 
 167         if (pcmk__same_node(node, replica->node)) {
 168             return true;
 169         }
 170     }
 171     return false;
 172 }
 173 
 174 /*!
 175  * \internal
 176  * \brief Get the container of a bundle's first replica
 177  *
 178  * \param[in] bundle  Bundle resource to get container for
 179  *
 180  * \return Container resource from first replica of \p bundle if any,
 181  *         otherwise NULL
 182  */
 183 pcmk_resource_t *
 184 pe__first_container(const pcmk_resource_t *bundle)
     /*  */
 185 {
 186     const pe__bundle_variant_data_t *bundle_data = NULL;
 187     const pcmk__bundle_replica_t *replica = NULL;
 188 
 189     get_bundle_variant_data(bundle_data, bundle);
 190     if (bundle_data->replicas == NULL) {
 191         return NULL;
 192     }
 193     replica = bundle_data->replicas->data;
 194     return replica->container;
 195 }
 196 
 197 /*!
 198  * \internal
 199  * \brief Iterate over bundle replicas
 200  *
 201  * \param[in,out] bundle     Bundle to iterate over
 202  * \param[in]     fn         Function to call for each replica (its return value
 203  *                           indicates whether to continue iterating)
 204  * \param[in,out] user_data  Pointer to pass to \p fn
 205  */
 206 void
 207 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
     /*  */
 208                            bool (*fn)(pcmk__bundle_replica_t *, void *),
 209                            void *user_data)
 210 {
 211     const pe__bundle_variant_data_t *bundle_data = NULL;
 212 
 213     get_bundle_variant_data(bundle_data, bundle);
 214     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 215         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
 216             break;
 217         }
 218     }
 219 }
 220 
 221 /*!
 222  * \internal
 223  * \brief Iterate over const bundle replicas
 224  *
 225  * \param[in]     bundle     Bundle to iterate over
 226  * \param[in]     fn         Function to call for each replica (its return value
 227  *                           indicates whether to continue iterating)
 228  * \param[in,out] user_data  Pointer to pass to \p fn
 229  */
 230 void
 231 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
     /*  */
 232                                  bool (*fn)(const pcmk__bundle_replica_t *,
 233                                             void *),
 234                                  void *user_data)
 235 {
 236     const pe__bundle_variant_data_t *bundle_data = NULL;
 237 
 238     get_bundle_variant_data(bundle_data, bundle);
 239     for (const GList *iter = bundle_data->replicas; iter != NULL;
 240          iter = iter->next) {
 241 
 242         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
 243             break;
 244         }
 245     }
 246 }
 247 
 248 static char *
 249 next_ip(const char *last_ip)
     /*  */
 250 {
 251     unsigned int oct1 = 0;
 252     unsigned int oct2 = 0;
 253     unsigned int oct3 = 0;
 254     unsigned int oct4 = 0;
 255     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 256 
 257     if (rc != 4) {
 258         /*@ TODO check for IPv6 */
 259         return NULL;
 260 
 261     } else if (oct3 > 253) {
 262         return NULL;
 263 
 264     } else if (oct4 > 253) {
 265         ++oct3;
 266         oct4 = 1;
 267 
 268     } else {
 269         ++oct4;
 270     }
 271 
 272     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 273 }
 274 
 275 static void
 276 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
     /*  */
 277             GString *buffer)
 278 {
 279     if(data->ip_range_start == NULL) {
 280         return;
 281 
 282     } else if(data->ip_last) {
 283         replica->ipaddr = next_ip(data->ip_last);
 284 
 285     } else {
 286         replica->ipaddr = strdup(data->ip_range_start);
 287     }
 288 
 289     data->ip_last = replica->ipaddr;
 290     switch (data->agent_type) {
 291         case PE__CONTAINER_AGENT_DOCKER:
 292         case PE__CONTAINER_AGENT_PODMAN:
 293             if (data->add_host) {
 294                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 295                                        data->prefix, replica->offset,
 296                                        replica->ipaddr);
 297             } else {
 298                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 299                                        replica->ipaddr, data->prefix,
 300                                        replica->offset);
 301             }
 302             break;
 303 
 304         default: // PE__CONTAINER_AGENT_UNKNOWN
 305             break;
 306     }
 307 }
 308 
 309 static xmlNode *
 310 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 311 {
 312     xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 313 
 314     crm_xml_add(rsc, PCMK_XA_ID, name);
 315     crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
 316     crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
 317     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 318 
 319     return rsc;
 320 }
 321 
 322 /*!
 323  * \internal
 324  * \brief Check whether cluster can manage resource inside container
 325  *
 326  * \param[in,out] data  Container variant data
 327  *
 328  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 329  *
 330  * \note The resource is manageable if an IP range or control port has been
 331  *       specified. If a control port is used without an IP range, replicas per
 332  *       host must be 1.
 333  */
 334 static bool
 335 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 336 {
 337     if(data->ip_range_start) {
 338         return TRUE;
 339     }
 340     if(data->control_port) {
 341         if(data->nreplicas_per_host > 1) {
 342             pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
 343                              "requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
 344                              data->prefix);
 345             data->nreplicas_per_host = 1;
 346             // @TODO to be sure:
 347             // pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
 348         }
 349         return TRUE;
 350     }
 351     return FALSE;
 352 }
 353 
 354 static int
 355 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 356                    pcmk__bundle_replica_t *replica)
 357 {
 358     if(data->ip_range_start) {
 359         char *id = NULL;
 360         xmlNode *xml_ip = NULL;
 361         xmlNode *xml_obj = NULL;
 362 
 363         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 364         pcmk__xml_sanitize_id(id);
 365         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 366         free(id);
 367 
 368         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
 369         pcmk__xe_set_id(xml_obj, "%s-attributes-%d",
 370                         data->prefix, replica->offset);
 371 
 372         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 373         if(data->host_network) {
 374             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 375         }
 376 
 377         if(data->host_netmask) {
 378             crm_create_nvpair_xml(xml_obj, NULL,
 379                                   "cidr_netmask", data->host_netmask);
 380 
 381         } else {
 382             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 383         }
 384 
 385         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
 386         crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
 387                           "60s", NULL);
 388 
 389         // TODO: Other ops? Timeouts and intervals from underlying resource?
 390 
 391         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 392                                 parent->priv->scheduler) != pcmk_rc_ok) {
 393             return pcmk_rc_unpack_error;
 394         }
 395 
 396         parent->priv->children = g_list_append(parent->priv->children,
 397                                                replica->ip);
 398     }
 399     return pcmk_rc_ok;
 400 }
 401 
 402 static const char*
 403 container_agent_str(enum pe__container_agent t)
     /*  */
 404 {
 405     switch (t) {
 406         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 407         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 408         default: // PE__CONTAINER_AGENT_UNKNOWN
 409             break;
 410     }
 411     return PE__CONTAINER_AGENT_UNKNOWN_S;
 412 }
 413 
 414 static int
 415 create_container_resource(pcmk_resource_t *parent,
     /*  */
 416                           const pe__bundle_variant_data_t *data,
 417                           pcmk__bundle_replica_t *replica)
 418 {
 419     char *id = NULL;
 420     xmlNode *xml_container = NULL;
 421     xmlNode *xml_obj = NULL;
 422 
 423     // Agent-specific
 424     const char *hostname_opt = NULL;
 425     const char *env_opt = NULL;
 426     const char *agent_str = NULL;
 427 
 428     GString *buffer = NULL;
 429     GString *dbuffer = NULL;
 430 
 431     // Where syntax differences are drop-in replacements, set them now
 432     switch (data->agent_type) {
 433         case PE__CONTAINER_AGENT_DOCKER:
 434         case PE__CONTAINER_AGENT_PODMAN:
 435             hostname_opt = "-h ";
 436             env_opt = "-e ";
 437             break;
 438         default:    // PE__CONTAINER_AGENT_UNKNOWN
 439             return pcmk_rc_unpack_error;
 440     }
 441     agent_str = container_agent_str(data->agent_type);
 442 
 443     buffer = g_string_sized_new(4096);
 444 
 445     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 446                            replica->offset);
 447     pcmk__xml_sanitize_id(id);
 448     xml_container = create_resource(id, "heartbeat", agent_str);
 449     free(id);
 450 
 451     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
 452     pcmk__xe_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 453 
 454     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 455     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
 456     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
 457     crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);
 458 
 459     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 460         g_string_append(buffer, " --restart=no");
 461     }
 462 
 463     /* Set a container hostname only if we have an IP to map it to. The user can
 464      * set -h or --uts=host themselves if they want a nicer name for logs, but
 465      * this makes applications happy who need their  hostname to match the IP
 466      * they bind to.
 467      */
 468     if (data->ip_range_start != NULL) {
 469         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 470                                replica->offset);
 471     }
 472     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 473 
 474     if (data->container_network != NULL) {
 475         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 476     }
 477 
 478     if (data->control_port != NULL) {
 479         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
 480                        data->control_port, NULL);
 481     } else {
 482         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
 483                                env_opt, DEFAULT_REMOTE_PORT);
 484     }
 485 
 486     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 487         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 488         char *source = NULL;
 489 
 490         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 491             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 492                                        replica->offset);
 493             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 494         }
 495 
 496         switch (data->agent_type) {
 497             case PE__CONTAINER_AGENT_DOCKER:
 498             case PE__CONTAINER_AGENT_PODMAN:
 499                 pcmk__g_strcat(buffer,
 500                                " -v ", pcmk__s(source, mount->source),
 501                                ":", mount->target, NULL);
 502 
 503                 if (mount->options != NULL) {
 504                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 505                 }
 506                 break;
 507             default:
 508                 break;
 509         }
 510         free(source);
 511     }
 512 
 513     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 514         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 515 
 516         switch (data->agent_type) {
 517             case PE__CONTAINER_AGENT_DOCKER:
 518             case PE__CONTAINER_AGENT_PODMAN:
 519                 if (replica->ipaddr != NULL) {
 520                     pcmk__g_strcat(buffer,
 521                                    " -p ", replica->ipaddr, ":", port->source,
 522                                    ":", port->target, NULL);
 523 
 524                 } else if (!pcmk__str_eq(data->container_network,
 525                                          PCMK_VALUE_HOST, pcmk__str_none)) {
 526                     // No need to do port mapping if net == host
 527                     pcmk__g_strcat(buffer,
 528                                    " -p ", port->source, ":", port->target,
 529                                    NULL);
 530                 }
 531                 break;
 532             default:
 533                 break;
 534         }
 535     }
 536 
 537     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 538      * it would cause restarts during rolling upgrades.
 539      *
 540      * In a previous version of the container resource creation logic, if
 541      * data->launcher_options is not NULL, we append
 542      * (" %s", data->launcher_options) even if data->launcher_options is an
 543      * empty string. Likewise for data->container_host_options. Using
 544      *
 545      *     pcmk__add_word(buffer, 0, data->launcher_options)
 546      *
 547      * removes that extra trailing space, causing a resource definition change.
 548      */
 549     if (data->launcher_options != NULL) {
 550         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 551     }
 552 
 553     if (data->container_host_options != NULL) {
 554         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 555     }
 556 
 557     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 558                           (const char *) buffer->str);
 559     g_string_free(buffer, TRUE);
 560 
 561     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 562                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 563     if (dbuffer != NULL) {
 564         g_string_free(dbuffer, TRUE);
 565     }
 566 
 567     if (replica->child != NULL) {
 568         if (data->container_command != NULL) {
 569             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 570                                   data->container_command);
 571         } else {
 572             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 573                                   SBIN_DIR "/" PCMK__SERVER_REMOTED);
 574         }
 575 
 576         /* TODO: Allow users to specify their own?
 577          *
 578          * We just want to know if the container is alive; we'll monitor the
 579          * child independently.
 580          */
 581         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 582 #if 0
 583         /* @TODO Consider supporting the use case where we can start and stop
 584          * resources, but not proxy local commands (such as setting node
 585          * attributes), by running the local executor in stand-alone mode.
 586          * However, this would probably be better done via ACLs as with other
 587          * Pacemaker Remote nodes.
 588          */
 589     } else if ((child != NULL) && data->untrusted) {
 590         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 591                               CRM_DAEMON_DIR "/" PCMK__SERVER_EXECD);
 592         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 593                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 594 #endif
 595     } else {
 596         if (data->container_command != NULL) {
 597             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 598                                   data->container_command);
 599         }
 600 
 601         /* TODO: Allow users to specify their own?
 602          *
 603          * We don't know what's in the container, so we just want to know if it
 604          * is alive.
 605          */
 606         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 607     }
 608 
 609     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
 610     crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
 611                       "60s", NULL);
 612 
 613     // TODO: Other ops? Timeouts and intervals from underlying resource?
 614     if (pe__unpack_resource(xml_container, &replica->container, parent,
 615                             parent->priv->scheduler) != pcmk_rc_ok) {
 616         return pcmk_rc_unpack_error;
 617     }
 618     pcmk__set_rsc_flags(replica->container, pcmk__rsc_replica_container);
 619     parent->priv->children = g_list_append(parent->priv->children,
 620                                            replica->container);
 621 
 622     return pcmk_rc_ok;
 623 }
 624 
 625 /*!
 626  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 627  *
 628  * \param[in,out] rsc    Resource to modify
 629  * \param[in]     uname  Name of node to ban
 630  */
 631 static void
 632 disallow_node(pcmk_resource_t *rsc, const char *uname)
     /*  */
 633 {
 634     gpointer match = g_hash_table_lookup(rsc->priv->allowed_nodes, uname);
 635 
 636     if (match) {
 637         ((pcmk_node_t *) match)->assign->score = -PCMK_SCORE_INFINITY;
 638         ((pcmk_node_t *) match)->assign->probe_mode = pcmk__probe_never;
 639     }
 640     g_list_foreach(rsc->priv->children, (GFunc) disallow_node,
 641                    (gpointer) uname);
 642 }
 643 
 644 static int
 645 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 646                        pcmk__bundle_replica_t *replica)
 647 {
 648     if (replica->child && valid_network(data)) {
 649         GHashTableIter gIter;
 650         pcmk_node_t *node = NULL;
 651         xmlNode *xml_remote = NULL;
 652         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 653         char *port_s = NULL;
 654         const char *uname = NULL;
 655         const char *connect_name = NULL;
 656         pcmk_scheduler_t *scheduler = parent->priv->scheduler;
 657 
 658         if (pe_find_resource(scheduler->priv->resources, id) != NULL) {
 659             free(id);
 660             // The biggest hammer we have
 661             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 662                                    replica->child->id, replica->offset);
 663             //@TODO return error instead of asserting?
 664             pcmk__assert(pe_find_resource(scheduler->priv->resources,
 665                                           id) == NULL);
 666         }
 667 
 668         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 669          * connection does not have its own IP is a magic string that we use to
 670          * support nested remotes (i.e. a bundle running on a remote node).
 671          */
 672         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 673 
 674         if (data->control_port == NULL) {
 675             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 676         }
 677 
 678         /* This sets replica->container as replica->remote's container, which is
 679          * similar to what happens with guest nodes. This is how the scheduler
 680          * knows that the bundle node is fenced by recovering the container, and
 681          * that remote should be ordered relative to the container.
 682          */
 683         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 684                                           NULL, NULL, NULL,
 685                                           connect_name, (data->control_port?
 686                                           data->control_port : port_s));
 687         free(port_s);
 688 
 689         /* Abandon our created ID, and pull the copy from the XML, because we
 690          * need something that will get freed during scheduler data cleanup to
 691          * use as the node ID and uname.
 692          */
 693         free(id);
 694         id = NULL;
 695         uname = pcmk__xe_id(xml_remote);
 696 
 697         /* Ensure a node has been created for the guest (it may have already
 698          * been, if it has a permanent node attribute), and ensure its weight is
 699          * -INFINITY so no other resources can run on it.
 700          */
 701         node = pcmk_find_node(scheduler, uname);
 702         if (node == NULL) {
 703             node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
 704                                   -PCMK_SCORE_INFINITY, scheduler);
 705         } else {
 706             node->assign->score = -PCMK_SCORE_INFINITY;
 707         }
 708         node->assign->probe_mode = pcmk__probe_never;
 709 
 710         /* unpack_remote_nodes() ensures that each remote node and guest node
 711          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
 712          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
 713          * obvious what nodes will be needed, so we do it just above.
 714          *
 715          * Worse, that means that the node may have been utilized while
 716          * unpacking other resources, without our weight correction. The most
 717          * likely place for this to happen is when pe__unpack_resource() calls
 718          * resource_location() to set a default score in symmetric clusters.
 719          * This adds a node *copy* to each resource's allowed nodes, and these
 720          * copies will have the wrong weight.
 721          *
 722          * As a hacky workaround, fix those copies here.
 723          *
 724          * @TODO Possible alternative: ensure bundles are unpacked before other
 725          * resources, so the weight is correct before any copies are made.
 726          */
 727         g_list_foreach(scheduler->priv->resources,
 728                        (GFunc) disallow_node, (gpointer) uname);
 729 
 730         replica->node = pe__copy_node(node);
 731         replica->node->assign->score = 500;
 732         replica->node->assign->probe_mode = pcmk__probe_exclusive;
 733 
 734         /* Ensure the node shows up as allowed and with the correct discovery set */
 735         if (replica->child->priv->allowed_nodes != NULL) {
 736             g_hash_table_destroy(replica->child->priv->allowed_nodes);
 737         }
 738         replica->child->priv->allowed_nodes =
 739             pcmk__strkey_table(NULL, pcmk__free_node_copy);
 740         g_hash_table_insert(replica->child->priv->allowed_nodes,
 741                             (gpointer) replica->node->priv->id,
 742                             pe__copy_node(replica->node));
 743 
 744         {
 745             const pcmk_resource_t *parent = replica->child->priv->parent;
 746             pcmk_node_t *copy = pe__copy_node(replica->node);
 747 
 748             copy->assign->score = -PCMK_SCORE_INFINITY;
 749             g_hash_table_insert(parent->priv->allowed_nodes,
 750                                 (gpointer) replica->node->priv->id, copy);
 751         }
 752         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 753                                 scheduler) != pcmk_rc_ok) {
 754             return pcmk_rc_unpack_error;
 755         }
 756 
 757         g_hash_table_iter_init(&gIter, replica->remote->priv->allowed_nodes);
 758         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 759             if (pcmk__is_pacemaker_remote_node(node)) {
 760                 /* Remote resources can only run on 'normal' cluster node */
 761                 node->assign->score = -PCMK_SCORE_INFINITY;
 762             }
 763         }
 764 
 765         replica->node->priv->remote = replica->remote;
 766 
 767         // Ensure pcmk__is_guest_or_bundle_node() functions correctly
 768         replica->remote->priv->launcher = replica->container;
 769 
 770         /* A bundle's #kind is closer to "container" (guest node) than the
 771          * "remote" set by pe_create_node().
 772          */
 773         pcmk__insert_dup(replica->node->priv->attrs,
 774                          CRM_ATTR_KIND, "container");
 775 
 776         /* One effect of this is that unpack_launcher() will add
 777          * replica->remote to replica->container's launched resources, which
 778          * will make pe__resource_contains_guest_node() true for
 779          * replica->container.
 780          *
 781          * replica->child does NOT get added to replica->container's launched
 782          * resources. The only noticeable effect if it did would be for its
 783          * fail count to be taken into account when checking
 784          * replica->container's migration threshold.
 785          */
 786         parent->priv->children = g_list_append(parent->priv->children,
 787                                                replica->remote);
 788     }
 789     return pcmk_rc_ok;
 790 }
 791 
 792 static int
 793 create_replica_resources(pcmk_resource_t *parent,
     /*  */
 794                          pe__bundle_variant_data_t *data,
 795                          pcmk__bundle_replica_t *replica)
 796 {
 797     int rc = pcmk_rc_ok;
 798 
 799     rc = create_container_resource(parent, data, replica);
 800     if (rc != pcmk_rc_ok) {
 801         return rc;
 802     }
 803 
 804     rc = create_ip_resource(parent, data, replica);
 805     if (rc != pcmk_rc_ok) {
 806         return rc;
 807     }
 808 
 809     rc = create_remote_resource(parent, data, replica);
 810     if (rc != pcmk_rc_ok) {
 811         return rc;
 812     }
 813 
 814     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 815         pcmk__insert_meta(replica->child->priv, "external-ip", replica->ipaddr);
 816     }
 817 
 818     if (replica->remote != NULL) {
 819         /*
 820          * Allow the remote connection resource to be allocated to a
 821          * different node than the one on which the container is active.
 822          *
 823          * This makes it possible to have Pacemaker Remote nodes running
 824          * containers with the remote executor inside in order to start
 825          * services inside those containers.
 826          */
 827         pcmk__set_rsc_flags(replica->remote, pcmk__rsc_remote_nesting_allowed);
 828     }
 829     return rc;
 830 }
 831 
 832 static void
 833 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 834           const char *target, const char *options, uint32_t flags)
 835 {
 836     pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
 837                                                    sizeof(pe__bundle_mount_t));
 838 
 839     mount->source = pcmk__str_copy(source);
 840     mount->target = pcmk__str_copy(target);
 841     mount->options = pcmk__str_copy(options);
 842     mount->flags = flags;
 843     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 844 }
 845 
 846 static void
 847 mount_free(pe__bundle_mount_t *mount)
     /*  */
 848 {
 849     free(mount->source);
 850     free(mount->target);
 851     free(mount->options);
 852     free(mount);
 853 }
 854 
 855 static void
 856 port_free(pe__bundle_port_t *port)
     /*  */
 857 {
 858     free(port->source);
 859     free(port->target);
 860     free(port);
 861 }
 862 
 863 static pcmk__bundle_replica_t *
 864 replica_for_remote(pcmk_resource_t *remote)
     /*  */
 865 {
 866     pcmk_resource_t *top = remote;
 867     pe__bundle_variant_data_t *bundle_data = NULL;
 868 
 869     if (top == NULL) {
 870         return NULL;
 871     }
 872     while (top->priv->parent != NULL) {
 873         top = top->priv->parent;
 874     }
 875 
 876     get_bundle_variant_data(bundle_data, top);
 877     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 878          gIter = gIter->next) {
 879         pcmk__bundle_replica_t *replica = gIter->data;
 880 
 881         if (replica->remote == remote) {
 882             return replica;
 883         }
 884     }
 885     CRM_LOG_ASSERT(FALSE);
 886     return NULL;
 887 }
 888 
 889 bool
 890 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
     /*  */
 891 {
 892     const char *value;
 893     GHashTable *params = NULL;
 894 
 895     if (rsc == NULL) {
 896         return false;
 897     }
 898 
 899     // Use NULL node since pcmk__bundle_expand() uses that to set value
 900     params = pe_rsc_params(rsc, NULL, rsc->priv->scheduler);
 901     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 902 
 903     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 904            && xml_contains_remote_node(rsc->priv->xml);
 905 }
 906 
 907 const char *
 908 pe__add_bundle_remote_name(pcmk_resource_t *rsc, xmlNode *xml,
     /*  */
 909                            const char *field)
 910 {
 911     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 912 
 913     pcmk_node_t *node = NULL;
 914     pcmk__bundle_replica_t *replica = NULL;
 915 
 916     if (!pe__bundle_needs_remote_name(rsc)) {
 917         return NULL;
 918     }
 919 
 920     replica = replica_for_remote(rsc);
 921     if (replica == NULL) {
 922         return NULL;
 923     }
 924 
 925     node = replica->container->priv->assigned_node;
 926     if (node == NULL) {
 927         /* If it won't be running anywhere after the
 928          * transition, go with where it's running now.
 929          */
 930         node = pcmk__current_node(replica->container);
 931     }
 932 
 933     if(node == NULL) {
 934         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 935         return NULL;
 936     }
 937 
 938     crm_trace("Setting address for bundle connection %s to bundle host %s",
 939               rsc->id, pcmk__node_name(node));
 940     if(xml != NULL && field != NULL) {
 941         crm_xml_add(xml, field, node->priv->name);
 942     }
 943 
 944     return node->priv->name;
 945 }
 946 
 947 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 948         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 949                                    "Bundle mount", pcmk__xe_id(mount_xml),  \
 950                                    flags, (flags_to_set), #flags_to_set);   \
 951     } while (0)
 952 
 953 gboolean
 954 pe__unpack_bundle(pcmk_resource_t *rsc, pcmk_scheduler_t *scheduler)
     /*  */
 955 {
 956     const char *value = NULL;
 957     xmlNode *xml_obj = NULL;
 958     const xmlNode *xml_child = NULL;
 959     xmlNode *xml_resource = NULL;
 960     pe__bundle_variant_data_t *bundle_data = NULL;
 961     bool need_log_mount = TRUE;
 962 
 963     pcmk__assert(rsc != NULL);
 964     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 965 
 966     bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
 967     rsc->priv->variant_opaque = bundle_data;
 968     bundle_data->prefix = strdup(rsc->id);
 969 
 970     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_DOCKER, NULL,
 971                                    NULL);
 972     if (xml_obj != NULL) {
 973         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
 974     }
 975 
 976     if (xml_obj == NULL) {
 977         xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PODMAN, NULL,
 978                                        NULL);
 979         if (xml_obj != NULL) {
 980             bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
 981         }
 982     }
 983 
 984     if (xml_obj == NULL) {
 985         return FALSE;
 986     }
 987 
 988     // Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
 989     value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
 990     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
 991 
 992     /* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
 993      * otherwise
 994      */
 995     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
 996     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
 997         bundle_data->nreplicas = bundle_data->promoted_max;
 998     } else {
 999         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1000     }
1001 
1002     /*
1003      * Communication between containers on the same host via the
1004      * floating IPs only works if the container is started with:
1005      *   --userland-proxy=false --ip-masq=false
1006      */
1007     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
1008     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1009     if (bundle_data->nreplicas_per_host == 1) {
1010         pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
1011     }
1012 
1013     bundle_data->container_command =
1014         crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
1015     bundle_data->launcher_options = crm_element_value_copy(xml_obj,
1016                                                            PCMK_XA_OPTIONS);
1017     bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
1018     bundle_data->container_network = crm_element_value_copy(xml_obj,
1019                                                             PCMK_XA_NETWORK);
1020 
1021     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_NETWORK, NULL,
1022                                    NULL);
1023     if(xml_obj) {
1024         bundle_data->ip_range_start =
1025             crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
1026         bundle_data->host_netmask =
1027             crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
1028         bundle_data->host_network =
1029             crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
1030         bundle_data->control_port =
1031             crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
1032         value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
1033         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1034             bundle_data->add_host = TRUE;
1035         }
1036 
1037         for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
1038                                               NULL, NULL);
1039              xml_child != NULL;
1040              xml_child = pcmk__xe_next(xml_child, PCMK_XE_PORT_MAPPING)) {
1041 
1042             pe__bundle_port_t *port =
1043                 pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1044 
1045             port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);
1046 
1047             if(port->source == NULL) {
1048                 port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
1049             } else {
1050                 port->target = crm_element_value_copy(xml_child,
1051                                                       PCMK_XA_INTERNAL_PORT);
1052             }
1053 
1054             if(port->source != NULL && strlen(port->source) > 0) {
1055                 if(port->target == NULL) {
1056                     port->target = strdup(port->source);
1057                 }
1058                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1059 
1060             } else {
1061                 pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
1062                                  pcmk__xe_id(xml_child));
1063                 port_free(port);
1064             }
1065         }
1066     }
1067 
1068     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_STORAGE, NULL,
1069                                    NULL);
1070     for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
1071                                           NULL, NULL);
1072          xml_child != NULL;
1073          xml_child = pcmk__xe_next(xml_child, PCMK_XE_STORAGE_MAPPING)) {
1074 
1075         const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
1076         const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
1077         const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
1078         int flags = pe__bundle_mount_none;
1079 
1080         if (source == NULL) {
1081             source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
1082             pe__set_bundle_mount_flags(xml_child, flags,
1083                                        pe__bundle_mount_subdir);
1084         }
1085 
1086         if (source && target) {
1087             mount_add(bundle_data, source, target, options, flags);
1088             if (strcmp(target, "/var/log") == 0) {
1089                 need_log_mount = FALSE;
1090             }
1091         } else {
1092             pcmk__config_err("Invalid mount directive %s",
1093                              pcmk__xe_id(xml_child));
1094         }
1095     }
1096 
1097     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PRIMITIVE, NULL,
1098                                    NULL);
1099     if (xml_obj && valid_network(bundle_data)) {
1100         const char *suffix = NULL;
1101         char *value = NULL;
1102         xmlNode *xml_set = NULL;
1103 
1104         xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);
1105 
1106         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1107          * part of the resource name, so that bundles don't restart in a rolling
1108          * upgrade. (It also avoids needing to change regression tests.)
1109          */
1110         suffix = (const char *) xml_resource->name;
1111         if (bundle_data->promoted_max > 0) {
1112             suffix = "master";
1113         }
1114 
1115         pcmk__xe_set_id(xml_resource, "%s-%s", bundle_data->prefix, suffix);
1116 
1117         xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
1118         pcmk__xe_set_id(xml_set, "%s-%s-meta",
1119                         bundle_data->prefix, xml_resource->name);
1120 
1121         crm_create_nvpair_xml(xml_set, NULL,
1122                               PCMK_META_ORDERED, PCMK_VALUE_TRUE);
1123 
1124         value = pcmk__itoa(bundle_data->nreplicas);
1125         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
1126         free(value);
1127 
1128         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1129         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
1130         free(value);
1131 
1132         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
1133                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1134 
1135         if (bundle_data->promoted_max) {
1136             crm_create_nvpair_xml(xml_set, NULL,
1137                                   PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);
1138 
1139             value = pcmk__itoa(bundle_data->promoted_max);
1140             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
1141             free(value);
1142         }
1143 
1144         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
1145         pcmk__xml_copy(xml_resource, xml_obj);
1146 
1147     } else if(xml_obj) {
1148         pcmk__config_err("Cannot control %s inside %s without either "
1149                          PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
1150                          rsc->id, pcmk__xe_id(xml_obj));
1151         return FALSE;
1152     }
1153 
1154     if(xml_resource) {
1155         int lpc = 0;
1156         GList *childIter = NULL;
1157         pe__bundle_port_t *port = NULL;
1158         GString *buffer = NULL;
1159 
1160         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
1161                                 scheduler) != pcmk_rc_ok) {
1162             return FALSE;
1163         }
1164 
1165         /* Currently, we always map the default authentication key location
1166          * into the same location inside the container.
1167          *
1168          * Ideally, we would respect the host's PCMK_authkey_location, but:
1169          * - it may be different on different nodes;
1170          * - the actual connection will do extra checking to make sure the key
1171          *   file exists and is readable, that we can't do here on the DC
1172          * - tools such as crm_resource and crm_simulate may not have the same
1173          *   environment variables as the cluster, causing operation digests to
1174          *   differ
1175          *
1176          * Always using the default location inside the container is fine,
1177          * because we control the pacemaker_remote environment, and it avoids
1178          * having to pass another environment variable to the container.
1179          *
1180          * @TODO A better solution may be to have only pacemaker_remote use the
1181          * environment variable, and have the cluster nodes use a new
1182          * cluster option for key location. This would introduce the limitation
1183          * of the location being the same on all cluster nodes, but that's
1184          * reasonable.
1185          */
1186         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1187                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1188 
1189         if (need_log_mount) {
1190             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1191                       pe__bundle_mount_subdir);
1192         }
1193 
1194         port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1195         if(bundle_data->control_port) {
1196             port->source = strdup(bundle_data->control_port);
1197         } else {
1198             /* If we wanted to respect PCMK_remote_port, we could use
1199              * crm_default_remote_port() here and elsewhere in this file instead
1200              * of DEFAULT_REMOTE_PORT.
1201              *
1202              * However, it gains nothing, since we control both the container
1203              * environment and the connection resource parameters, and the user
1204              * can use a different port if desired by setting
1205              * PCMK_XA_CONTROL_PORT.
1206              */
1207             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1208         }
1209         port->target = strdup(port->source);
1210         bundle_data->ports = g_list_append(bundle_data->ports, port);
1211 
1212         buffer = g_string_sized_new(1024);
1213         for (childIter = bundle_data->child->priv->children;
1214              childIter != NULL; childIter = childIter->next) {
1215 
1216             pcmk__bundle_replica_t *replica = NULL;
1217 
1218             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1219             replica->child = childIter->data;
1220             pcmk__set_rsc_flags(replica->child, pcmk__rsc_exclusive_probes);
1221             replica->offset = lpc++;
1222 
1223             // Ensure the child's notify gets set based on the underlying primitive's value
1224             if (pcmk_is_set(replica->child->flags, pcmk__rsc_notify)) {
1225                 pcmk__set_rsc_flags(bundle_data->child, pcmk__rsc_notify);
1226             }
1227 
1228             allocate_ip(bundle_data, replica, buffer);
1229             bundle_data->replicas = g_list_append(bundle_data->replicas,
1230                                                   replica);
1231             bundle_data->attribute_target =
1232                 g_hash_table_lookup(replica->child->priv->meta,
1233                                     PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
1234         }
1235         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1236 
1237         if (bundle_data->attribute_target) {
1238             pcmk__insert_dup(rsc->priv->meta,
1239                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1240                              bundle_data->attribute_target);
1241             pcmk__insert_dup(bundle_data->child->priv->meta,
1242                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1243                              bundle_data->attribute_target);
1244         }
1245 
1246     } else {
1247         // Just a naked container, no pacemaker-remote
1248         GString *buffer = g_string_sized_new(1024);
1249 
1250         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1251             pcmk__bundle_replica_t *replica = NULL;
1252 
1253             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1254             replica->offset = lpc;
1255             allocate_ip(bundle_data, replica, buffer);
1256             bundle_data->replicas = g_list_append(bundle_data->replicas,
1257                                                   replica);
1258         }
1259         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1260     }
1261 
1262     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1263          gIter = gIter->next) {
1264         pcmk__bundle_replica_t *replica = gIter->data;
1265 
1266         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1267             pcmk__config_err("Failed unpacking resource %s", rsc->id);
1268             rsc->priv->fns->free(rsc);
1269             return FALSE;
1270         }
1271 
1272         /* Utilization needs special handling for bundles. It makes no sense for
1273          * the inner primitive to have utilization, because it is tied
1274          * one-to-one to the guest node created by the container resource -- and
1275          * there's no way to set capacities for that guest node anyway.
1276          *
1277          * What the user really wants is to configure utilization for the
1278          * container. However, the schema only allows utilization for
1279          * primitives, and the container resource is implicit anyway, so the
1280          * user can *only* configure utilization for the inner primitive. If
1281          * they do, move the primitive's utilization values to the container.
1282          *
1283          * @TODO This means that bundles without an inner primitive can't have
1284          * utilization. An alternative might be to allow utilization values in
1285          * the top-level bundle XML in the schema, and copy those to each
1286          * container.
1287          */
1288         if (replica->child != NULL) {
1289             GHashTable *empty = replica->container->priv->utilization;
1290 
1291             replica->container->priv->utilization =
1292                 replica->child->priv->utilization;
1293 
1294             replica->child->priv->utilization = empty;
1295         }
1296     }
1297 
1298     if (bundle_data->child) {
1299         rsc->priv->children = g_list_append(rsc->priv->children,
1300                                             bundle_data->child);
1301     }
1302     return TRUE;
1303 }
1304 
1305 static int
1306 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1307 {
1308     if (rsc) {
1309         gboolean child_active = rsc->priv->fns->active(rsc, all);
1310 
1311         if (child_active && !all) {
1312             return TRUE;
1313         } else if (!child_active && all) {
1314             return FALSE;
1315         }
1316     }
1317     return -1;
1318 }
1319 
1320 gboolean
1321 pe__bundle_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1322 {
1323     pe__bundle_variant_data_t *bundle_data = NULL;
1324     GList *iter = NULL;
1325 
1326     get_bundle_variant_data(bundle_data, rsc);
1327     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1328         pcmk__bundle_replica_t *replica = iter->data;
1329         int rsc_active;
1330 
1331         rsc_active = replica_resource_active(replica->ip, all);
1332         if (rsc_active >= 0) {
1333             return (gboolean) rsc_active;
1334         }
1335 
1336         rsc_active = replica_resource_active(replica->child, all);
1337         if (rsc_active >= 0) {
1338             return (gboolean) rsc_active;
1339         }
1340 
1341         rsc_active = replica_resource_active(replica->container, all);
1342         if (rsc_active >= 0) {
1343             return (gboolean) rsc_active;
1344         }
1345 
1346         rsc_active = replica_resource_active(replica->remote, all);
1347         if (rsc_active >= 0) {
1348             return (gboolean) rsc_active;
1349         }
1350     }
1351 
1352     /* If "all" is TRUE, we've already checked that no resources were inactive,
1353      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1354      * so return FALSE.
1355      */
1356     return all;
1357 }
1358 
1359 /*!
1360  * \internal
1361  * \brief Find the bundle replica corresponding to a given node
1362  *
1363  * \param[in] bundle  Top-level bundle resource
1364  * \param[in] node    Node to search for
1365  *
1366  * \return Bundle replica if found, NULL otherwise
1367  */
1368 pcmk_resource_t *
1369 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
     /*  */
1370 {
1371     pe__bundle_variant_data_t *bundle_data = NULL;
1372 
1373     pcmk__assert((bundle != NULL) && (node != NULL));
1374 
1375     get_bundle_variant_data(bundle_data, bundle);
1376     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1377          gIter = gIter->next) {
1378         pcmk__bundle_replica_t *replica = gIter->data;
1379 
1380         pcmk__assert((replica != NULL) && (replica->node != NULL));
1381         if (pcmk__same_node(replica->node, node)) {
1382             return replica->child;
1383         }
1384     }
1385     return NULL;
1386 }
1387 
1388 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1389                   "GList *")
1390 int
1391 pe__bundle_xml(pcmk__output_t *out, va_list args)
1392 {
1393     uint32_t show_opts = va_arg(args, uint32_t);
1394     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1395     GList *only_node = va_arg(args, GList *);
1396     GList *only_rsc = va_arg(args, GList *);
1397 
1398     pe__bundle_variant_data_t *bundle_data = NULL;
1399     int rc = pcmk_rc_no_output;
1400     gboolean printed_header = FALSE;
1401     gboolean print_everything = TRUE;
1402 
1403     const char *desc = NULL;
1404 
1405     pcmk__assert(rsc != NULL);
1406     get_bundle_variant_data(bundle_data, rsc);
1407 
1408     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
1409         return rc;
1410     }
1411 
1412     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1413 
1414     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1415          gIter = gIter->next) {
1416         pcmk__bundle_replica_t *replica = gIter->data;
1417         pcmk_resource_t *ip = replica->ip;
1418         pcmk_resource_t *child = replica->child;
1419         pcmk_resource_t *container = replica->container;
1420         pcmk_resource_t *remote = replica->remote;
1421         char *id = NULL;
1422         gboolean print_ip, print_child, print_ctnr, print_remote;
1423 
1424         pcmk__assert(replica != NULL);
1425 
1426         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1427             continue;
1428         }
1429 
1430         print_ip = (ip != NULL)
1431                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1432                                                   print_everything);
1433         print_child = (child != NULL)
1434                       && !child->priv->fns->is_filtered(child, only_rsc,
1435                                                         print_everything);
1436         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1437                                                         print_everything);
1438         print_remote = (remote != NULL)
1439                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1440                                                           print_everything);
1441 
1442         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1443             continue;
1444         }
1445 
1446         if (!printed_header) {
1447             const char *type = container_agent_str(bundle_data->agent_type);
1448             const char *unique = pcmk__flag_text(rsc->flags, pcmk__rsc_unique);
1449             const char *maintenance = pcmk__flag_text(rsc->flags,
1450                                                       pcmk__rsc_maintenance);
1451             const char *managed = pcmk__flag_text(rsc->flags,
1452                                                   pcmk__rsc_managed);
1453             const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);
1454 
1455             printed_header = TRUE;
1456 
1457             desc = pe__resource_description(rsc, show_opts);
1458 
1459             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
1460                                           PCMK_XA_ID, rsc->id,
1461                                           PCMK_XA_TYPE, type,
1462                                           PCMK_XA_IMAGE, bundle_data->image,
1463                                           PCMK_XA_UNIQUE, unique,
1464                                           PCMK_XA_MAINTENANCE, maintenance,
1465                                           PCMK_XA_MANAGED, managed,
1466                                           PCMK_XA_FAILED, failed,
1467                                           PCMK_XA_DESCRIPTION, desc,
1468                                           NULL);
1469             pcmk__assert(rc == pcmk_rc_ok);
1470         }
1471 
1472         id = pcmk__itoa(replica->offset);
1473         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
1474                                       PCMK_XA_ID, id,
1475                                       NULL);
1476         free(id);
1477         pcmk__assert(rc == pcmk_rc_ok);
1478 
1479         if (print_ip) {
1480             out->message(out, (const char *) ip->priv->xml->name, show_opts,
1481                          ip, only_node, only_rsc);
1482         }
1483 
1484         if (print_child) {
1485             out->message(out, (const char *) child->priv->xml->name,
1486                          show_opts, child, only_node, only_rsc);
1487         }
1488 
1489         if (print_ctnr) {
1490             out->message(out, (const char *) container->priv->xml->name,
1491                          show_opts, container, only_node, only_rsc);
1492         }
1493 
1494         if (print_remote) {
1495             out->message(out, (const char *) remote->priv->xml->name,
1496                          show_opts, remote, only_node, only_rsc);
1497         }
1498 
1499         pcmk__output_xml_pop_parent(out); // replica
1500     }
1501 
1502     if (printed_header) {
1503         pcmk__output_xml_pop_parent(out); // bundle
1504     }
1505 
1506     return rc;
1507 }
1508 
1509 static void
1510 pe__bundle_replica_output_html(pcmk__output_t *out,
     /*  */
1511                                pcmk__bundle_replica_t *replica,
1512                                pcmk_node_t *node, uint32_t show_opts)
1513 {
1514     pcmk_resource_t *rsc = replica->child;
1515 
1516     int offset = 0;
1517     char buffer[LINE_MAX];
1518 
1519     if(rsc == NULL) {
1520         rsc = replica->container;
1521     }
1522 
1523     if (replica->remote) {
1524         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1525                            rsc_printable_id(replica->remote));
1526     } else {
1527         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1528                            rsc_printable_id(replica->container));
1529     }
1530     if (replica->ipaddr) {
1531         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1532                            replica->ipaddr);
1533     }
1534 
1535     pe__common_output_html(out, rsc, buffer, node, show_opts);
1536 }
1537 
1538 /*!
1539  * \internal
1540  * \brief Get a string describing a resource's unmanaged state or lack thereof
1541  *
1542  * \param[in] rsc  Resource to describe
1543  *
1544  * \return A string indicating that a resource is in maintenance mode or
1545  *         otherwise unmanaged, or an empty string otherwise
1546  */
1547 static const char *
1548 get_unmanaged_str(const pcmk_resource_t *rsc)
     /*  */
1549 {
1550     if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
1551         return " (maintenance)";
1552     }
1553     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
1554         return " (unmanaged)";
1555     }
1556     return "";
1557 }
1558 
1559 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1560                   "GList *")
1561 int
1562 pe__bundle_html(pcmk__output_t *out, va_list args)
1563 {
1564     uint32_t show_opts = va_arg(args, uint32_t);
1565     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1566     GList *only_node = va_arg(args, GList *);
1567     GList *only_rsc = va_arg(args, GList *);
1568 
1569     const char *desc = NULL;
1570     pe__bundle_variant_data_t *bundle_data = NULL;
1571     int rc = pcmk_rc_no_output;
1572     gboolean print_everything = TRUE;
1573 
1574     pcmk__assert(rsc != NULL);
1575     get_bundle_variant_data(bundle_data, rsc);
1576 
1577     desc = pe__resource_description(rsc, show_opts);
1578 
1579     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
1580         return rc;
1581     }
1582 
1583     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1584 
1585     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1586          gIter = gIter->next) {
1587         pcmk__bundle_replica_t *replica = gIter->data;
1588         pcmk_resource_t *ip = replica->ip;
1589         pcmk_resource_t *child = replica->child;
1590         pcmk_resource_t *container = replica->container;
1591         pcmk_resource_t *remote = replica->remote;
1592         gboolean print_ip, print_child, print_ctnr, print_remote;
1593 
1594         pcmk__assert(replica != NULL);
1595 
1596         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1597             continue;
1598         }
1599 
1600         print_ip = (ip != NULL)
1601                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1602                                                   print_everything);
1603         print_child = (child != NULL)
1604                       && !child->priv->fns->is_filtered(child, only_rsc,
1605                                                         print_everything);
1606         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1607                                                         print_everything);
1608         print_remote = (remote != NULL)
1609                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1610                                                           print_everything);
1611 
1612         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1613             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1614             /* The text output messages used below require pe_print_implicit to
1615              * be set to do anything.
1616              */
1617             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1618 
1619             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1620                                      (bundle_data->nreplicas > 1)? " set" : "",
1621                                      rsc->id, bundle_data->image,
1622                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1623                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1624                                      get_unmanaged_str(rsc));
1625 
1626             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1627                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1628             }
1629 
1630             if (print_ip) {
1631                 out->message(out, (const char *) ip->priv->xml->name,
1632                              new_show_opts, ip, only_node, only_rsc);
1633             }
1634 
1635             if (print_child) {
1636                 out->message(out, (const char *) child->priv->xml->name,
1637                              new_show_opts, child, only_node, only_rsc);
1638             }
1639 
1640             if (print_ctnr) {
1641                 out->message(out, (const char *) container->priv->xml->name,
1642                              new_show_opts, container, only_node, only_rsc);
1643             }
1644 
1645             if (print_remote) {
1646                 out->message(out, (const char *) remote->priv->xml->name,
1647                              new_show_opts, remote, only_node, only_rsc);
1648             }
1649 
1650             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1651                 out->end_list(out);
1652             }
1653         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1654             continue;
1655         } else {
1656             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1657                                      (bundle_data->nreplicas > 1)? " set" : "",
1658                                      rsc->id, bundle_data->image,
1659                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1660                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1661                                      get_unmanaged_str(rsc));
1662 
1663             pe__bundle_replica_output_html(out, replica,
1664                                            pcmk__current_node(container),
1665                                            show_opts);
1666         }
1667     }
1668 
1669     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1670     return rc;
1671 }
1672 
1673 static void
1674 pe__bundle_replica_output_text(pcmk__output_t *out,
     /*  */
1675                                pcmk__bundle_replica_t *replica,
1676                                pcmk_node_t *node, uint32_t show_opts)
1677 {
1678     const pcmk_resource_t *rsc = replica->child;
1679 
1680     int offset = 0;
1681     char buffer[LINE_MAX];
1682 
1683     if(rsc == NULL) {
1684         rsc = replica->container;
1685     }
1686 
1687     if (replica->remote) {
1688         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1689                            rsc_printable_id(replica->remote));
1690     } else {
1691         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1692                            rsc_printable_id(replica->container));
1693     }
1694     if (replica->ipaddr) {
1695         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1696                            replica->ipaddr);
1697     }
1698 
1699     pe__common_output_text(out, rsc, buffer, node, show_opts);
1700 }
1701 
1702 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1703                   "GList *")
1704 int
1705 pe__bundle_text(pcmk__output_t *out, va_list args)
1706 {
1707     uint32_t show_opts = va_arg(args, uint32_t);
1708     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1709     GList *only_node = va_arg(args, GList *);
1710     GList *only_rsc = va_arg(args, GList *);
1711 
1712     const char *desc = NULL;
1713     pe__bundle_variant_data_t *bundle_data = NULL;
1714     int rc = pcmk_rc_no_output;
1715     gboolean print_everything = TRUE;
1716 
1717     desc = pe__resource_description(rsc, show_opts);
1718 
1719     pcmk__assert(rsc != NULL);
1720     get_bundle_variant_data(bundle_data, rsc);
1721 
1722     if (rsc->priv->fns->is_filtered(rsc, only_rsc, TRUE)) {
1723         return rc;
1724     }
1725 
1726     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1727 
1728     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1729          gIter = gIter->next) {
1730         pcmk__bundle_replica_t *replica = gIter->data;
1731         pcmk_resource_t *ip = replica->ip;
1732         pcmk_resource_t *child = replica->child;
1733         pcmk_resource_t *container = replica->container;
1734         pcmk_resource_t *remote = replica->remote;
1735         gboolean print_ip, print_child, print_ctnr, print_remote;
1736 
1737         pcmk__assert(replica != NULL);
1738 
1739         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1740             continue;
1741         }
1742 
1743         print_ip = (ip != NULL)
1744                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1745                                                   print_everything);
1746         print_child = (child != NULL)
1747                       && !child->priv->fns->is_filtered(child, only_rsc,
1748                                                         print_everything);
1749         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1750                                                         print_everything);
1751         print_remote = (remote != NULL)
1752                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1753                                                           print_everything);
1754 
1755         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1756             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1757             /* The text output messages used below require pe_print_implicit to
1758              * be set to do anything.
1759              */
1760             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1761 
1762             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1763                                      (bundle_data->nreplicas > 1)? " set" : "",
1764                                      rsc->id, bundle_data->image,
1765                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1766                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1767                                      get_unmanaged_str(rsc));
1768 
1769             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1770                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1771             }
1772 
1773             out->begin_list(out, NULL, NULL, NULL);
1774 
1775             if (print_ip) {
1776                 out->message(out, (const char *) ip->priv->xml->name,
1777                              new_show_opts, ip, only_node, only_rsc);
1778             }
1779 
1780             if (print_child) {
1781                 out->message(out, (const char *) child->priv->xml->name,
1782                              new_show_opts, child, only_node, only_rsc);
1783             }
1784 
1785             if (print_ctnr) {
1786                 out->message(out, (const char *) container->priv->xml->name,
1787                              new_show_opts, container, only_node, only_rsc);
1788             }
1789 
1790             if (print_remote) {
1791                 out->message(out, (const char *) remote->priv->xml->name,
1792                              new_show_opts, remote, only_node, only_rsc);
1793             }
1794 
1795             out->end_list(out);
1796         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1797             continue;
1798         } else {
1799             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1800                                      (bundle_data->nreplicas > 1)? " set" : "",
1801                                      rsc->id, bundle_data->image,
1802                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1803                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1804                                      get_unmanaged_str(rsc));
1805 
1806             pe__bundle_replica_output_text(out, replica,
1807                                            pcmk__current_node(container),
1808                                            show_opts);
1809         }
1810     }
1811 
1812     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1813     return rc;
1814 }
1815 
1816 static void
1817 free_bundle_replica(pcmk__bundle_replica_t *replica)
     /*  */
1818 {
1819     if (replica == NULL) {
1820         return;
1821     }
1822 
1823     pcmk__free_node_copy(replica->node);
1824     replica->node = NULL;
1825 
1826     if (replica->ip) {
1827         pcmk__xml_free(replica->ip->priv->xml);
1828         replica->ip->priv->xml = NULL;
1829         replica->ip->priv->fns->free(replica->ip);
1830     }
1831     if (replica->container) {
1832         pcmk__xml_free(replica->container->priv->xml);
1833         replica->container->priv->xml = NULL;
1834         replica->container->priv->fns->free(replica->container);
1835     }
1836     if (replica->remote) {
1837         pcmk__xml_free(replica->remote->priv->xml);
1838         replica->remote->priv->xml = NULL;
1839         replica->remote->priv->fns->free(replica->remote);
1840     }
1841     free(replica->ipaddr);
1842     free(replica);
1843 }
1844 
1845 void
1846 pe__free_bundle(pcmk_resource_t *rsc)
     /*  */
1847 {
1848     pe__bundle_variant_data_t *bundle_data = NULL;
1849     CRM_CHECK(rsc != NULL, return);
1850 
1851     get_bundle_variant_data(bundle_data, rsc);
1852     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
1853 
1854     free(bundle_data->prefix);
1855     free(bundle_data->image);
1856     free(bundle_data->control_port);
1857     free(bundle_data->host_network);
1858     free(bundle_data->host_netmask);
1859     free(bundle_data->ip_range_start);
1860     free(bundle_data->container_network);
1861     free(bundle_data->launcher_options);
1862     free(bundle_data->container_command);
1863     g_free(bundle_data->container_host_options);
1864 
1865     g_list_free_full(bundle_data->replicas,
1866                      (GDestroyNotify) free_bundle_replica);
1867     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1868     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1869     g_list_free(rsc->priv->children);
1870 
1871     if(bundle_data->child) {
1872         pcmk__xml_free(bundle_data->child->priv->xml);
1873         bundle_data->child->priv->xml = NULL;
1874         bundle_data->child->priv->fns->free(bundle_data->child);
1875     }
1876     common_free(rsc);
1877 }
1878 
1879 enum rsc_role_e
1880 pe__bundle_resource_state(const pcmk_resource_t *rsc, gboolean current)
     /*  */
1881 {
1882     enum rsc_role_e container_role = pcmk_role_unknown;
1883     return container_role;
1884 }
1885 
1886 /*!
1887  * \brief Get the number of configured replicas in a bundle
1888  *
1889  * \param[in] rsc  Bundle resource
1890  *
1891  * \return Number of configured replicas, or 0 on error
1892  */
1893 int
1894 pe_bundle_replicas(const pcmk_resource_t *rsc)
     /*  */
1895 {
1896     if (pcmk__is_bundle(rsc)) {
1897         pe__bundle_variant_data_t *bundle_data = NULL;
1898 
1899         get_bundle_variant_data(bundle_data, rsc);
1900         return bundle_data->nreplicas;
1901     }
1902     return 0;
1903 }
1904 
1905 void
1906 pe__count_bundle(pcmk_resource_t *rsc)
     /*  */
1907 {
1908     pe__bundle_variant_data_t *bundle_data = NULL;
1909 
1910     get_bundle_variant_data(bundle_data, rsc);
1911     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
1912         pcmk__bundle_replica_t *replica = item->data;
1913 
1914         if (replica->ip) {
1915             replica->ip->priv->fns->count(replica->ip);
1916         }
1917         if (replica->child) {
1918             replica->child->priv->fns->count(replica->child);
1919         }
1920         if (replica->container) {
1921             replica->container->priv->fns->count(replica->container);
1922         }
1923         if (replica->remote) {
1924             replica->remote->priv->fns->count(replica->remote);
1925         }
1926     }
1927 }
1928 
1929 gboolean
1930 pe__bundle_is_filtered(const pcmk_resource_t *rsc, GList *only_rsc,
     /*  */
1931                        gboolean check_parent)
1932 {
1933     gboolean passes = FALSE;
1934     pe__bundle_variant_data_t *bundle_data = NULL;
1935 
1936     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
1937         passes = TRUE;
1938     } else {
1939         get_bundle_variant_data(bundle_data, rsc);
1940 
1941         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
1942             pcmk__bundle_replica_t *replica = gIter->data;
1943             pcmk_resource_t *ip = replica->ip;
1944             pcmk_resource_t *child = replica->child;
1945             pcmk_resource_t *container = replica->container;
1946             pcmk_resource_t *remote = replica->remote;
1947 
1948             if ((ip != NULL)
1949                 && !ip->priv->fns->is_filtered(ip, only_rsc, FALSE)) {
1950                 passes = TRUE;
1951                 break;
1952             }
1953             if ((child != NULL)
1954                 && !child->priv->fns->is_filtered(child, only_rsc, FALSE)) {
1955                 passes = TRUE;
1956                 break;
1957             }
1958             if (!container->priv->fns->is_filtered(container, only_rsc,
1959                                                    FALSE)) {
1960                 passes = TRUE;
1961                 break;
1962             }
1963             if ((remote != NULL)
1964                 && !remote->priv->fns->is_filtered(remote, only_rsc, FALSE)) {
1965                 passes = TRUE;
1966                 break;
1967             }
1968         }
1969     }
1970 
1971     return !passes;
1972 }
1973 
1974 /*!
1975  * \internal
1976  * \brief Get a list of a bundle's containers
1977  *
1978  * \param[in] bundle  Bundle resource
1979  *
1980  * \return Newly created list of \p bundle's containers
1981  * \note It is the caller's responsibility to free the result with
1982  *       g_list_free().
1983  */
1984 GList *
1985 pe__bundle_containers(const pcmk_resource_t *bundle)
     /*  */
1986 {
1987     GList *containers = NULL;
1988     const pe__bundle_variant_data_t *data = NULL;
1989 
1990     get_bundle_variant_data(data, bundle);
1991     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
1992         pcmk__bundle_replica_t *replica = iter->data;
1993 
1994         containers = g_list_append(containers, replica->container);
1995     }
1996     return containers;
1997 }
1998 
1999 // Bundle implementation of pcmk__rsc_methods_t:active_node()
2000 pcmk_node_t *
2001 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
     /*  */
2002                        unsigned int *count_clean)
2003 {
2004     pcmk_node_t *active = NULL;
2005     pcmk_node_t *node = NULL;
2006     pcmk_resource_t *container = NULL;
2007     GList *containers = NULL;
2008     GList *iter = NULL;
2009     GHashTable *nodes = NULL;
2010     const pe__bundle_variant_data_t *data = NULL;
2011 
2012     if (count_all != NULL) {
2013         *count_all = 0;
2014     }
2015     if (count_clean != NULL) {
2016         *count_clean = 0;
2017     }
2018     if (rsc == NULL) {
2019         return NULL;
2020     }
2021 
2022     /* For the purposes of this method, we only care about where the bundle's
2023      * containers are active, so build a list of active containers.
2024      */
2025     get_bundle_variant_data(data, rsc);
2026     for (iter = data->replicas; iter != NULL; iter = iter->next) {
2027         pcmk__bundle_replica_t *replica = iter->data;
2028 
2029         if (replica->container->priv->active_nodes != NULL) {
2030             containers = g_list_append(containers, replica->container);
2031         }
2032     }
2033     if (containers == NULL) {
2034         return NULL;
2035     }
2036 
2037     /* If the bundle has only a single active container, just use that
2038      * container's method. If live migration is ever supported for bundle
2039      * containers, this will allow us to prefer the migration source when there
2040      * is only one container and it is migrating. For now, this just lets us
2041      * avoid creating the nodes table.
2042      */
2043     if (pcmk__list_of_1(containers)) {
2044         container = containers->data;
2045         node = container->priv->fns->active_node(container, count_all,
2046                                                  count_clean);
2047         g_list_free(containers);
2048         return node;
2049     }
2050 
2051     // Add all containers' active nodes to a hash table (for uniqueness)
2052     nodes = g_hash_table_new(NULL, NULL);
2053     for (iter = containers; iter != NULL; iter = iter->next) {
2054         container = iter->data;
2055         for (GList *node_iter = container->priv->active_nodes;
2056              node_iter != NULL; node_iter = node_iter->next) {
2057 
2058             node = node_iter->data;
2059 
2060             // If insert returns true, we haven't counted this node yet
2061             if (g_hash_table_insert(nodes, (gpointer) node->details,
2062                                     (gpointer) node)
2063                 && !pe__count_active_node(rsc, node, &active, count_all,
2064                                           count_clean)) {
2065                 goto done;
2066             }
2067         }
2068     }
2069 
2070 done:
2071     g_list_free(containers);
2072     g_hash_table_destroy(nodes);
2073     return active;
2074 }
2075 
2076 /*!
2077  * \internal
2078  * \brief Get maximum bundle resource instances per node
2079  *
2080  * \param[in] rsc  Bundle resource to check
2081  *
2082  * \return Maximum number of \p rsc instances that can be active on one node
2083  */
2084 unsigned int
2085 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
     /*  */
2086 {
2087     pe__bundle_variant_data_t *bundle_data = NULL;
2088 
2089     get_bundle_variant_data(bundle_data, rsc);
2090     pcmk__assert(bundle_data->nreplicas_per_host >= 0);
2091     return (unsigned int) bundle_data->nreplicas_per_host;
2092 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
