root/lib/pengine/bundle.c

/* */

DEFINITIONS

1. pe__bundle_max
2. pe__bundled_resource
3. pe__get_rsc_in_container
4. pe__node_is_bundle_instance
5. pe__first_container
6. pe__foreach_bundle_replica
7. pe__foreach_const_bundle_replica
8. next_ip
9. allocate_ip
10. create_resource
11. valid_network
12. create_ip_resource
13. container_agent_str
14. create_container_resource
15. disallow_node
16. create_remote_resource
17. create_replica_resources
18. mount_add
19. mount_free
20. port_free
21. replica_for_remote
22. pe__bundle_needs_remote_name
23. pe__add_bundle_remote_name
24. pe__unpack_bundle
25. replica_resource_active
26. pe__bundle_active
27. pe__find_bundle_replica
28. PCMK__OUTPUT_ARGS
29. pe__bundle_replica_output_html
30. get_unmanaged_str
31. PCMK__OUTPUT_ARGS
32. pe__bundle_replica_output_text
33. PCMK__OUTPUT_ARGS
34. free_bundle_replica
35. pe__free_bundle
36. pe__bundle_resource_state
37. pe_bundle_replicas
38. pe__count_bundle
39. pe__bundle_is_filtered
40. pe__bundle_containers
41. pe__bundle_active_node
42. pe__bundle_max_per_node

   1 /*
   2  * Copyright 2004-2025 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdbool.h>                    // bool, true, false
  14 #include <stdint.h>
  15 
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/common/xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 enum pe__bundle_mount_flags {
  24     pe__bundle_mount_none       = 0x00,
  25 
  26     // mount instance-specific subdirectory rather than source directly
  27     pe__bundle_mount_subdir     = 0x01
  28 };
  29 
  30 typedef struct {
  31     char *source;
  32     char *target;
  33     char *options;
  34     uint32_t flags; // bitmask of pe__bundle_mount_flags
  35 } pe__bundle_mount_t;
  36 
  37 typedef struct {
  38     char *source;
  39     char *target;
  40 } pe__bundle_port_t;
  41 
  42 enum pe__container_agent {
  43     PE__CONTAINER_AGENT_UNKNOWN,
  44     PE__CONTAINER_AGENT_DOCKER,
  45     PE__CONTAINER_AGENT_PODMAN,
  46 };
  47 
  48 #define PE__CONTAINER_AGENT_UNKNOWN_S "unknown"
  49 #define PE__CONTAINER_AGENT_DOCKER_S  "docker"
  50 #define PE__CONTAINER_AGENT_PODMAN_S  "podman"
  51 
  52 typedef struct pe__bundle_variant_data_s {
  53         int promoted_max;
  54         int nreplicas;
  55         int nreplicas_per_host;
  56         char *prefix;
  57         char *image;
  58         const char *ip_last;
  59         char *host_network;
  60         char *host_netmask;
  61         char *control_port;
  62         char *container_network;
  63         char *ip_range_start;
  64         gboolean add_host;
  65         gchar *container_host_options;
  66         char *container_command;
  67         char *launcher_options;
  68         const char *attribute_target;
  69 
  70         pcmk_resource_t *child;
  71 
  72         GList *replicas;    // pcmk__bundle_replica_t *
  73         GList *ports;       // pe__bundle_port_t *
  74         GList *mounts;      // pe__bundle_mount_t *
  75 
  76         /* @TODO Maybe use a more object-oriented design instead, with a set of
  77          * methods that are different per type rather than switching on this
  78          */
  79         enum pe__container_agent agent_type;
  80 } pe__bundle_variant_data_t;
  81 
  82 #define get_bundle_variant_data(data, rsc) do { \
  83         pcmk__assert(pcmk__is_bundle(rsc));     \
  84         data = rsc->priv->variant_opaque;       \
  85     } while (0)
  86 
  87 /*!
  88  * \internal
  89  * \brief Get maximum number of bundle replicas allowed to run
  90  *
  91  * \param[in] rsc  Bundle or bundled resource to check
  92  *
  93  * \return Maximum replicas for bundle corresponding to \p rsc
  94  */
  95 int
  96 pe__bundle_max(const pcmk_resource_t *rsc)
     /*  */
  97 {
  98     const pe__bundle_variant_data_t *bundle_data = NULL;
  99 
 100     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 101     return bundle_data->nreplicas;
 102 }
 103 
 104 /*!
 105  * \internal
 106  * \brief Get the resource inside a bundle
 107  *
 108  * \param[in] bundle  Bundle to check
 109  *
 110  * \return Resource inside \p bundle if any, otherwise NULL
 111  */
 112 pcmk_resource_t *
 113 pe__bundled_resource(const pcmk_resource_t *rsc)
     /*  */
 114 {
 115     const pe__bundle_variant_data_t *bundle_data = NULL;
 116 
 117     get_bundle_variant_data(bundle_data, pe__const_top_resource(rsc, true));
 118     return bundle_data->child;
 119 }
 120 
 121 /*!
 122  * \internal
 123  * \brief Get containerized resource corresponding to a given bundle container
 124  *
 125  * \param[in] instance  Collective instance that might be a bundle container
 126  *
 127  * \return Bundled resource instance inside \p instance if it is a bundle
 128  *         container instance, otherwise NULL
 129  */
 130 const pcmk_resource_t *
 131 pe__get_rsc_in_container(const pcmk_resource_t *instance)
     /*  */
 132 {
 133     const pe__bundle_variant_data_t *data = NULL;
 134     const pcmk_resource_t *top = pe__const_top_resource(instance, true);
 135 
 136     if (!pcmk__is_bundle(top)) {
 137         return NULL;
 138     }
 139     get_bundle_variant_data(data, top);
 140 
 141     for (const GList *iter = data->replicas; iter != NULL; iter = iter->next) {
 142         const pcmk__bundle_replica_t *replica = iter->data;
 143 
 144         if (instance == replica->container) {
 145             return replica->child;
 146         }
 147     }
 148     return NULL;
 149 }
 150 
 151 /*!
 152  * \internal
 153  * \brief Check whether a given node is created by a bundle
 154  *
 155  * \param[in] bundle  Bundle resource to check
 156  * \param[in] node    Node to check
 157  *
 158  * \return true if \p node is an instance of \p bundle, otherwise false
 159  */
 160 bool
 161 pe__node_is_bundle_instance(const pcmk_resource_t *bundle,
     /*  */
 162                             const pcmk_node_t *node)
 163 {
 164     pe__bundle_variant_data_t *bundle_data = NULL;
 165 
 166     get_bundle_variant_data(bundle_data, bundle);
 167     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 168         pcmk__bundle_replica_t *replica = iter->data;
 169 
 170         if (pcmk__same_node(node, replica->node)) {
 171             return true;
 172         }
 173     }
 174     return false;
 175 }
 176 
 177 /*!
 178  * \internal
 179  * \brief Get the container of a bundle's first replica
 180  *
 181  * \param[in] bundle  Bundle resource to get container for
 182  *
 183  * \return Container resource from first replica of \p bundle if any,
 184  *         otherwise NULL
 185  */
 186 pcmk_resource_t *
 187 pe__first_container(const pcmk_resource_t *bundle)
     /*  */
 188 {
 189     const pe__bundle_variant_data_t *bundle_data = NULL;
 190     const pcmk__bundle_replica_t *replica = NULL;
 191 
 192     get_bundle_variant_data(bundle_data, bundle);
 193     if (bundle_data->replicas == NULL) {
 194         return NULL;
 195     }
 196     replica = bundle_data->replicas->data;
 197     return replica->container;
 198 }
 199 
 200 /*!
 201  * \internal
 202  * \brief Iterate over bundle replicas
 203  *
 204  * \param[in,out] bundle     Bundle to iterate over
 205  * \param[in]     fn         Function to call for each replica (its return value
 206  *                           indicates whether to continue iterating)
 207  * \param[in,out] user_data  Pointer to pass to \p fn
 208  */
 209 void
 210 pe__foreach_bundle_replica(pcmk_resource_t *bundle,
     /*  */
 211                            bool (*fn)(pcmk__bundle_replica_t *, void *),
 212                            void *user_data)
 213 {
 214     const pe__bundle_variant_data_t *bundle_data = NULL;
 215 
 216     get_bundle_variant_data(bundle_data, bundle);
 217     for (GList *iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
 218         if (!fn((pcmk__bundle_replica_t *) iter->data, user_data)) {
 219             break;
 220         }
 221     }
 222 }
 223 
 224 /*!
 225  * \internal
 226  * \brief Iterate over const bundle replicas
 227  *
 228  * \param[in]     bundle     Bundle to iterate over
 229  * \param[in]     fn         Function to call for each replica (its return value
 230  *                           indicates whether to continue iterating)
 231  * \param[in,out] user_data  Pointer to pass to \p fn
 232  */
 233 void
 234 pe__foreach_const_bundle_replica(const pcmk_resource_t *bundle,
     /*  */
 235                                  bool (*fn)(const pcmk__bundle_replica_t *,
 236                                             void *),
 237                                  void *user_data)
 238 {
 239     const pe__bundle_variant_data_t *bundle_data = NULL;
 240 
 241     get_bundle_variant_data(bundle_data, bundle);
 242     for (const GList *iter = bundle_data->replicas; iter != NULL;
 243          iter = iter->next) {
 244 
 245         if (!fn((const pcmk__bundle_replica_t *) iter->data, user_data)) {
 246             break;
 247         }
 248     }
 249 }
 250 
 251 static char *
 252 next_ip(const char *last_ip)
     /*  */
 253 {
 254     unsigned int oct1 = 0;
 255     unsigned int oct2 = 0;
 256     unsigned int oct3 = 0;
 257     unsigned int oct4 = 0;
 258     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
 259 
 260     if (rc != 4) {
 261         /*@ TODO check for IPv6 */
 262         return NULL;
 263 
 264     } else if (oct3 > 253) {
 265         return NULL;
 266 
 267     } else if (oct4 > 253) {
 268         ++oct3;
 269         oct4 = 1;
 270 
 271     } else {
 272         ++oct4;
 273     }
 274 
 275     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
 276 }
 277 
 278 static void
 279 allocate_ip(pe__bundle_variant_data_t *data, pcmk__bundle_replica_t *replica,
     /*  */
 280             GString *buffer)
 281 {
 282     if(data->ip_range_start == NULL) {
 283         return;
 284 
 285     } else if(data->ip_last) {
 286         replica->ipaddr = next_ip(data->ip_last);
 287 
 288     } else {
 289         replica->ipaddr = strdup(data->ip_range_start);
 290     }
 291 
 292     data->ip_last = replica->ipaddr;
 293     switch (data->agent_type) {
 294         case PE__CONTAINER_AGENT_DOCKER:
 295         case PE__CONTAINER_AGENT_PODMAN:
 296             if (data->add_host) {
 297                 g_string_append_printf(buffer, " --add-host=%s-%d:%s",
 298                                        data->prefix, replica->offset,
 299                                        replica->ipaddr);
 300             } else {
 301                 g_string_append_printf(buffer, " --hosts-entry=%s=%s-%d",
 302                                        replica->ipaddr, data->prefix,
 303                                        replica->offset);
 304             }
 305             break;
 306 
 307         default: // PE__CONTAINER_AGENT_UNKNOWN
 308             break;
 309     }
 310 }
 311 
 312 static xmlNode *
 313 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
 314 {
 315     xmlNode *rsc = pcmk__xe_create(NULL, PCMK_XE_PRIMITIVE);
 316 
 317     crm_xml_add(rsc, PCMK_XA_ID, name);
 318     crm_xml_add(rsc, PCMK_XA_CLASS, PCMK_RESOURCE_CLASS_OCF);
 319     crm_xml_add(rsc, PCMK_XA_PROVIDER, provider);
 320     crm_xml_add(rsc, PCMK_XA_TYPE, kind);
 321 
 322     return rsc;
 323 }
 324 
 325 /*!
 326  * \internal
 327  * \brief Check whether cluster can manage resource inside container
 328  *
 329  * \param[in,out] data  Container variant data
 330  *
 331  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 332  *
 333  * \note The resource is manageable if an IP range or control port has been
 334  *       specified. If a control port is used without an IP range, replicas per
 335  *       host must be 1.
 336  */
 337 static bool
 338 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 339 {
 340     if(data->ip_range_start) {
 341         return TRUE;
 342     }
 343     if(data->control_port) {
 344         if(data->nreplicas_per_host > 1) {
 345             pcmk__config_err("Specifying the '" PCMK_XA_CONTROL_PORT "' for %s "
 346                              "requires '" PCMK_XA_REPLICAS_PER_HOST "=1'",
 347                              data->prefix);
 348             data->nreplicas_per_host = 1;
 349             // @TODO to be sure:
 350             // pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
 351         }
 352         return TRUE;
 353     }
 354     return FALSE;
 355 }
 356 
 357 static int
 358 create_ip_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 359                    pcmk__bundle_replica_t *replica)
 360 {
 361     if(data->ip_range_start) {
 362         char *id = NULL;
 363         xmlNode *xml_ip = NULL;
 364         xmlNode *xml_obj = NULL;
 365 
 366         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 367         pcmk__xml_sanitize_id(id);
 368         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 369         free(id);
 370 
 371         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_INSTANCE_ATTRIBUTES);
 372         pcmk__xe_set_id(xml_obj, "%s-attributes-%d",
 373                         data->prefix, replica->offset);
 374 
 375         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 376         if(data->host_network) {
 377             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 378         }
 379 
 380         if(data->host_netmask) {
 381             crm_create_nvpair_xml(xml_obj, NULL,
 382                                   "cidr_netmask", data->host_netmask);
 383 
 384         } else {
 385             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 386         }
 387 
 388         xml_obj = pcmk__xe_create(xml_ip, PCMK_XE_OPERATIONS);
 389         crm_create_op_xml(xml_obj, pcmk__xe_id(xml_ip), PCMK_ACTION_MONITOR,
 390                           "60s", NULL);
 391 
 392         // TODO: Other ops? Timeouts and intervals from underlying resource?
 393 
 394         if (pe__unpack_resource(xml_ip, &replica->ip, parent,
 395                                 parent->priv->scheduler) != pcmk_rc_ok) {
 396             return pcmk_rc_unpack_error;
 397         }
 398 
 399         parent->priv->children = g_list_append(parent->priv->children,
 400                                                replica->ip);
 401     }
 402     return pcmk_rc_ok;
 403 }
 404 
 405 static const char*
 406 container_agent_str(enum pe__container_agent t)
     /*  */
 407 {
 408     switch (t) {
 409         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
 410         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
 411         default: // PE__CONTAINER_AGENT_UNKNOWN
 412             break;
 413     }
 414     return PE__CONTAINER_AGENT_UNKNOWN_S;
 415 }
 416 
 417 static int
 418 create_container_resource(pcmk_resource_t *parent,
     /*  */
 419                           const pe__bundle_variant_data_t *data,
 420                           pcmk__bundle_replica_t *replica)
 421 {
 422     char *id = NULL;
 423     xmlNode *xml_container = NULL;
 424     xmlNode *xml_obj = NULL;
 425 
 426     // Agent-specific
 427     const char *hostname_opt = NULL;
 428     const char *env_opt = NULL;
 429     const char *agent_str = NULL;
 430 
 431     GString *buffer = NULL;
 432     GString *dbuffer = NULL;
 433 
 434     // Where syntax differences are drop-in replacements, set them now
 435     switch (data->agent_type) {
 436         case PE__CONTAINER_AGENT_DOCKER:
 437         case PE__CONTAINER_AGENT_PODMAN:
 438             hostname_opt = "-h ";
 439             env_opt = "-e ";
 440             break;
 441         default:    // PE__CONTAINER_AGENT_UNKNOWN
 442             return pcmk_rc_unpack_error;
 443     }
 444     agent_str = container_agent_str(data->agent_type);
 445 
 446     buffer = g_string_sized_new(4096);
 447 
 448     id = crm_strdup_printf("%s-%s-%d", data->prefix, agent_str,
 449                            replica->offset);
 450     pcmk__xml_sanitize_id(id);
 451     xml_container = create_resource(id, "heartbeat", agent_str);
 452     free(id);
 453 
 454     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_INSTANCE_ATTRIBUTES);
 455     pcmk__xe_set_id(xml_obj, "%s-attributes-%d", data->prefix, replica->offset);
 456 
 457     crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 458     crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", PCMK_VALUE_TRUE);
 459     crm_create_nvpair_xml(xml_obj, NULL, "force_kill", PCMK_VALUE_FALSE);
 460     crm_create_nvpair_xml(xml_obj, NULL, "reuse", PCMK_VALUE_FALSE);
 461 
 462     if (data->agent_type == PE__CONTAINER_AGENT_DOCKER) {
 463         g_string_append(buffer, " --restart=no");
 464     }
 465 
 466     /* Set a container hostname only if we have an IP to map it to. The user can
 467      * set -h or --uts=host themselves if they want a nicer name for logs, but
 468      * this makes applications happy who need their  hostname to match the IP
 469      * they bind to.
 470      */
 471     if (data->ip_range_start != NULL) {
 472         g_string_append_printf(buffer, " %s%s-%d", hostname_opt, data->prefix,
 473                                replica->offset);
 474     }
 475     pcmk__g_strcat(buffer, " ", env_opt, "PCMK_stderr=1", NULL);
 476 
 477     if (data->container_network != NULL) {
 478         pcmk__g_strcat(buffer, " --net=", data->container_network, NULL);
 479     }
 480 
 481     if (data->control_port != NULL) {
 482         pcmk__g_strcat(buffer, " ", env_opt, "PCMK_" PCMK__ENV_REMOTE_PORT "=",
 483                        data->control_port, NULL);
 484     } else {
 485         g_string_append_printf(buffer, " %sPCMK_" PCMK__ENV_REMOTE_PORT "=%d",
 486                                env_opt, DEFAULT_REMOTE_PORT);
 487     }
 488 
 489     for (GList *iter = data->mounts; iter != NULL; iter = iter->next) {
 490         pe__bundle_mount_t *mount = (pe__bundle_mount_t *) iter->data;
 491         char *source = NULL;
 492 
 493         if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 494             source = crm_strdup_printf("%s/%s-%d", mount->source, data->prefix,
 495                                        replica->offset);
 496             pcmk__add_separated_word(&dbuffer, 1024, source, ",");
 497         }
 498 
 499         switch (data->agent_type) {
 500             case PE__CONTAINER_AGENT_DOCKER:
 501             case PE__CONTAINER_AGENT_PODMAN:
 502                 pcmk__g_strcat(buffer,
 503                                " -v ", pcmk__s(source, mount->source),
 504                                ":", mount->target, NULL);
 505 
 506                 if (mount->options != NULL) {
 507                     pcmk__g_strcat(buffer, ":", mount->options, NULL);
 508                 }
 509                 break;
 510             default:
 511                 break;
 512         }
 513         free(source);
 514     }
 515 
 516     for (GList *iter = data->ports; iter != NULL; iter = iter->next) {
 517         pe__bundle_port_t *port = (pe__bundle_port_t *) iter->data;
 518 
 519         switch (data->agent_type) {
 520             case PE__CONTAINER_AGENT_DOCKER:
 521             case PE__CONTAINER_AGENT_PODMAN:
 522                 if (replica->ipaddr != NULL) {
 523                     pcmk__g_strcat(buffer,
 524                                    " -p ", replica->ipaddr, ":", port->source,
 525                                    ":", port->target, NULL);
 526 
 527                 } else if (!pcmk__str_eq(data->container_network,
 528                                          PCMK_VALUE_HOST, pcmk__str_none)) {
 529                     // No need to do port mapping if net == host
 530                     pcmk__g_strcat(buffer,
 531                                    " -p ", port->source, ":", port->target,
 532                                    NULL);
 533                 }
 534                 break;
 535             default:
 536                 break;
 537         }
 538     }
 539 
 540     /* @COMPAT: We should use pcmk__add_word() here, but we can't yet, because
 541      * it would cause restarts during rolling upgrades.
 542      *
 543      * In a previous version of the container resource creation logic, if
 544      * data->launcher_options is not NULL, we append
 545      * (" %s", data->launcher_options) even if data->launcher_options is an
 546      * empty string. Likewise for data->container_host_options. Using
 547      *
 548      *     pcmk__add_word(buffer, 0, data->launcher_options)
 549      *
 550      * removes that extra trailing space, causing a resource definition change.
 551      */
 552     if (data->launcher_options != NULL) {
 553         pcmk__g_strcat(buffer, " ", data->launcher_options, NULL);
 554     }
 555 
 556     if (data->container_host_options != NULL) {
 557         pcmk__g_strcat(buffer, " ", data->container_host_options, NULL);
 558     }
 559 
 560     crm_create_nvpair_xml(xml_obj, NULL, "run_opts",
 561                           (const char *) buffer->str);
 562     g_string_free(buffer, TRUE);
 563 
 564     crm_create_nvpair_xml(xml_obj, NULL, "mount_points",
 565                           (dbuffer != NULL)? (const char *) dbuffer->str : "");
 566     if (dbuffer != NULL) {
 567         g_string_free(dbuffer, TRUE);
 568     }
 569 
 570     if (replica->child != NULL) {
 571         if (data->container_command != NULL) {
 572             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 573                                   data->container_command);
 574         } else {
 575             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 576                                   SBIN_DIR "/" PCMK__SERVER_REMOTED);
 577         }
 578 
 579         /* TODO: Allow users to specify their own?
 580          *
 581          * We just want to know if the container is alive; we'll monitor the
 582          * child independently.
 583          */
 584         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 585 #if 0
 586         /* @TODO Consider supporting the use case where we can start and stop
 587          * resources, but not proxy local commands (such as setting node
 588          * attributes), by running the local executor in stand-alone mode.
 589          * However, this would probably be better done via ACLs as with other
 590          * Pacemaker Remote nodes.
 591          */
 592     } else if ((child != NULL) && data->untrusted) {
 593         crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 594                               CRM_DAEMON_DIR "/" PCMK__SERVER_EXECD);
 595         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 596                               CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 597 #endif
 598     } else {
 599         if (data->container_command != NULL) {
 600             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 601                                   data->container_command);
 602         }
 603 
 604         /* TODO: Allow users to specify their own?
 605          *
 606          * We don't know what's in the container, so we just want to know if it
 607          * is alive.
 608          */
 609         crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 610     }
 611 
 612     xml_obj = pcmk__xe_create(xml_container, PCMK_XE_OPERATIONS);
 613     crm_create_op_xml(xml_obj, pcmk__xe_id(xml_container), PCMK_ACTION_MONITOR,
 614                       "60s", NULL);
 615 
 616     // TODO: Other ops? Timeouts and intervals from underlying resource?
 617     if (pe__unpack_resource(xml_container, &replica->container, parent,
 618                             parent->priv->scheduler) != pcmk_rc_ok) {
 619         return pcmk_rc_unpack_error;
 620     }
 621     pcmk__set_rsc_flags(replica->container, pcmk__rsc_replica_container);
 622     parent->priv->children = g_list_append(parent->priv->children,
 623                                            replica->container);
 624 
 625     return pcmk_rc_ok;
 626 }
 627 
 628 /*!
 629  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 630  *
 631  * \param[in,out] rsc    Resource to modify
 632  * \param[in]     uname  Name of node to ban
 633  */
 634 static void
 635 disallow_node(pcmk_resource_t *rsc, const char *uname)
     /*  */
 636 {
 637     gpointer match = g_hash_table_lookup(rsc->priv->allowed_nodes, uname);
 638 
 639     if (match) {
 640         ((pcmk_node_t *) match)->assign->score = -PCMK_SCORE_INFINITY;
 641         ((pcmk_node_t *) match)->assign->probe_mode = pcmk__probe_never;
 642     }
 643     g_list_foreach(rsc->priv->children, (GFunc) disallow_node,
 644                    (gpointer) uname);
 645 }
 646 
 647 static int
 648 create_remote_resource(pcmk_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 649                        pcmk__bundle_replica_t *replica)
 650 {
 651     if (replica->child && valid_network(data)) {
 652         GHashTableIter gIter;
 653         pcmk_node_t *node = NULL;
 654         xmlNode *xml_remote = NULL;
 655         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 656         char *port_s = NULL;
 657         const char *uname = NULL;
 658         const char *connect_name = NULL;
 659         pcmk_scheduler_t *scheduler = parent->priv->scheduler;
 660 
 661         if (pe_find_resource(scheduler->priv->resources, id) != NULL) {
 662             free(id);
 663             // The biggest hammer we have
 664             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 665                                    replica->child->id, replica->offset);
 666             //@TODO return error instead of asserting?
 667             pcmk__assert(pe_find_resource(scheduler->priv->resources,
 668                                           id) == NULL);
 669         }
 670 
 671         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 672          * connection does not have its own IP is a magic string that we use to
 673          * support nested remotes (i.e. a bundle running on a remote node).
 674          */
 675         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 676 
 677         if (data->control_port == NULL) {
 678             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 679         }
 680 
 681         /* This sets replica->container as replica->remote's container, which is
 682          * similar to what happens with guest nodes. This is how the scheduler
 683          * knows that the bundle node is fenced by recovering the container, and
 684          * that remote should be ordered relative to the container.
 685          */
 686         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 687                                           NULL, NULL, NULL,
 688                                           connect_name, (data->control_port?
 689                                           data->control_port : port_s));
 690         free(port_s);
 691 
 692         /* Abandon our created ID, and pull the copy from the XML, because we
 693          * need something that will get freed during scheduler data cleanup to
 694          * use as the node ID and uname.
 695          */
 696         free(id);
 697         id = NULL;
 698         uname = pcmk__xe_id(xml_remote);
 699 
 700         /* Ensure a node has been created for the guest (it may have already
 701          * been, if it has a permanent node attribute), and ensure its weight is
 702          * -INFINITY so no other resources can run on it.
 703          */
 704         node = pcmk_find_node(scheduler, uname);
 705         if (node == NULL) {
 706             node = pe_create_node(uname, uname, PCMK_VALUE_REMOTE,
 707                                   -PCMK_SCORE_INFINITY, scheduler);
 708         } else {
 709             node->assign->score = -PCMK_SCORE_INFINITY;
 710         }
 711         node->assign->probe_mode = pcmk__probe_never;
 712 
 713         /* unpack_remote_nodes() ensures that each remote node and guest node
 714          * has a pcmk_node_t entry. Ideally, it would do the same for bundle
 715          * nodes. Unfortunately, a bundle has to be mostly unpacked before it's
 716          * obvious what nodes will be needed, so we do it just above.
 717          *
 718          * Worse, that means that the node may have been utilized while
 719          * unpacking other resources, without our weight correction. The most
 720          * likely place for this to happen is when pe__unpack_resource() calls
 721          * resource_location() to set a default score in symmetric clusters.
 722          * This adds a node *copy* to each resource's allowed nodes, and these
 723          * copies will have the wrong weight.
 724          *
 725          * As a hacky workaround, fix those copies here.
 726          *
 727          * @TODO Possible alternative: ensure bundles are unpacked before other
 728          * resources, so the weight is correct before any copies are made.
 729          */
 730         g_list_foreach(scheduler->priv->resources,
 731                        (GFunc) disallow_node, (gpointer) uname);
 732 
 733         replica->node = pe__copy_node(node);
 734         replica->node->assign->score = 500;
 735         replica->node->assign->probe_mode = pcmk__probe_exclusive;
 736 
 737         /* Ensure the node shows up as allowed and with the correct discovery set */
 738         if (replica->child->priv->allowed_nodes != NULL) {
 739             g_hash_table_destroy(replica->child->priv->allowed_nodes);
 740         }
 741         replica->child->priv->allowed_nodes =
 742             pcmk__strkey_table(NULL, pcmk__free_node_copy);
 743         g_hash_table_insert(replica->child->priv->allowed_nodes,
 744                             (gpointer) replica->node->priv->id,
 745                             pe__copy_node(replica->node));
 746 
 747         {
 748             const pcmk_resource_t *parent = replica->child->priv->parent;
 749             pcmk_node_t *copy = pe__copy_node(replica->node);
 750 
 751             copy->assign->score = -PCMK_SCORE_INFINITY;
 752             g_hash_table_insert(parent->priv->allowed_nodes,
 753                                 (gpointer) replica->node->priv->id, copy);
 754         }
 755         if (pe__unpack_resource(xml_remote, &replica->remote, parent,
 756                                 scheduler) != pcmk_rc_ok) {
 757             return pcmk_rc_unpack_error;
 758         }
 759 
 760         g_hash_table_iter_init(&gIter, replica->remote->priv->allowed_nodes);
 761         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 762             if (pcmk__is_pacemaker_remote_node(node)) {
 763                 /* Remote resources can only run on 'normal' cluster node */
 764                 node->assign->score = -PCMK_SCORE_INFINITY;
 765             }
 766         }
 767 
 768         replica->node->priv->remote = replica->remote;
 769 
 770         // Ensure pcmk__is_guest_or_bundle_node() functions correctly
 771         replica->remote->priv->launcher = replica->container;
 772 
 773         /* A bundle's #kind is closer to "container" (guest node) than the
 774          * "remote" set by pe_create_node().
 775          */
 776         pcmk__insert_dup(replica->node->priv->attrs,
 777                          CRM_ATTR_KIND, "container");
 778 
 779         /* One effect of this is that unpack_launcher() will add
 780          * replica->remote to replica->container's launched resources, which
 781          * will make pe__resource_contains_guest_node() true for
 782          * replica->container.
 783          *
 784          * replica->child does NOT get added to replica->container's launched
 785          * resources. The only noticeable effect if it did would be for its
 786          * fail count to be taken into account when checking
 787          * replica->container's migration threshold.
 788          */
 789         parent->priv->children = g_list_append(parent->priv->children,
 790                                                replica->remote);
 791     }
 792     return pcmk_rc_ok;
 793 }
 794 
 795 static int
 796 create_replica_resources(pcmk_resource_t *parent,
     /*  */
 797                          pe__bundle_variant_data_t *data,
 798                          pcmk__bundle_replica_t *replica)
 799 {
 800     int rc = pcmk_rc_ok;
 801 
 802     rc = create_container_resource(parent, data, replica);
 803     if (rc != pcmk_rc_ok) {
 804         return rc;
 805     }
 806 
 807     rc = create_ip_resource(parent, data, replica);
 808     if (rc != pcmk_rc_ok) {
 809         return rc;
 810     }
 811 
 812     rc = create_remote_resource(parent, data, replica);
 813     if (rc != pcmk_rc_ok) {
 814         return rc;
 815     }
 816 
 817     if ((replica->child != NULL) && (replica->ipaddr != NULL)) {
 818         pcmk__insert_meta(replica->child->priv, "external-ip", replica->ipaddr);
 819     }
 820 
 821     if (replica->remote != NULL) {
 822         /*
 823          * Allow the remote connection resource to be allocated to a
 824          * different node than the one on which the container is active.
 825          *
 826          * This makes it possible to have Pacemaker Remote nodes running
 827          * containers with the remote executor inside in order to start
 828          * services inside those containers.
 829          */
 830         pcmk__set_rsc_flags(replica->remote, pcmk__rsc_remote_nesting_allowed);
 831     }
 832     return rc;
 833 }
 834 
 835 static void
 836 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 837           const char *target, const char *options, uint32_t flags)
 838 {
 839     pe__bundle_mount_t *mount = pcmk__assert_alloc(1,
 840                                                    sizeof(pe__bundle_mount_t));
 841 
 842     mount->source = pcmk__str_copy(source);
 843     mount->target = pcmk__str_copy(target);
 844     mount->options = pcmk__str_copy(options);
 845     mount->flags = flags;
 846     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 847 }
 848 
 849 static void
 850 mount_free(pe__bundle_mount_t *mount)
     /*  */
 851 {
 852     free(mount->source);
 853     free(mount->target);
 854     free(mount->options);
 855     free(mount);
 856 }
 857 
 858 static void
 859 port_free(pe__bundle_port_t *port)
     /*  */
 860 {
 861     free(port->source);
 862     free(port->target);
 863     free(port);
 864 }
 865 
 866 static pcmk__bundle_replica_t *
 867 replica_for_remote(pcmk_resource_t *remote)
     /*  */
 868 {
 869     pcmk_resource_t *top = remote;
 870     pe__bundle_variant_data_t *bundle_data = NULL;
 871 
 872     if (top == NULL) {
 873         return NULL;
 874     }
 875     while (top->priv->parent != NULL) {
 876         top = top->priv->parent;
 877     }
 878 
 879     get_bundle_variant_data(bundle_data, top);
 880     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 881          gIter = gIter->next) {
 882         pcmk__bundle_replica_t *replica = gIter->data;
 883 
 884         if (replica->remote == remote) {
 885             return replica;
 886         }
 887     }
 888     CRM_LOG_ASSERT(FALSE);
 889     return NULL;
 890 }
 891 
 892 bool
 893 pe__bundle_needs_remote_name(pcmk_resource_t *rsc)
     /*  */
 894 {
 895     const char *value;
 896     GHashTable *params = NULL;
 897 
 898     if (rsc == NULL) {
 899         return false;
 900     }
 901 
 902     // Use NULL node since pcmk__bundle_expand() uses that to set value
 903     params = pe_rsc_params(rsc, NULL, rsc->priv->scheduler);
 904     value = g_hash_table_lookup(params, PCMK_REMOTE_RA_ADDR);
 905 
 906     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 907            && xml_contains_remote_node(rsc->priv->xml);
 908 }
 909 
 910 const char *
 911 pe__add_bundle_remote_name(pcmk_resource_t *rsc, xmlNode *xml,
     /*  */
 912                            const char *field)
 913 {
 914     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 915 
 916     pcmk_node_t *node = NULL;
 917     pcmk__bundle_replica_t *replica = NULL;
 918 
 919     if (!pe__bundle_needs_remote_name(rsc)) {
 920         return NULL;
 921     }
 922 
 923     replica = replica_for_remote(rsc);
 924     if (replica == NULL) {
 925         return NULL;
 926     }
 927 
 928     node = replica->container->priv->assigned_node;
 929     if (node == NULL) {
 930         /* If it won't be running anywhere after the
 931          * transition, go with where it's running now.
 932          */
 933         node = pcmk__current_node(replica->container);
 934     }
 935 
 936     if(node == NULL) {
 937         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 938         return NULL;
 939     }
 940 
 941     crm_trace("Setting address for bundle connection %s to bundle host %s",
 942               rsc->id, pcmk__node_name(node));
 943     if(xml != NULL && field != NULL) {
 944         crm_xml_add(xml, field, node->priv->name);
 945     }
 946 
 947     return node->priv->name;
 948 }
 949 
 950 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
 951         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
 952                                    "Bundle mount", pcmk__xe_id(mount_xml),  \
 953                                    flags, (flags_to_set), #flags_to_set);   \
 954     } while (0)
 955 
 956 bool
 957 pe__unpack_bundle(pcmk_resource_t *rsc)
     /*  */
 958 {
 959     const char *value = NULL;
 960     xmlNode *xml_obj = NULL;
 961     const xmlNode *xml_child = NULL;
 962     xmlNode *xml_resource = NULL;
 963     pe__bundle_variant_data_t *bundle_data = NULL;
 964     bool need_log_mount = TRUE;
 965 
 966     pcmk__assert(rsc != NULL);
 967     pcmk__rsc_trace(rsc, "Processing resource %s...", rsc->id);
 968 
 969     bundle_data = pcmk__assert_alloc(1, sizeof(pe__bundle_variant_data_t));
 970     rsc->priv->variant_opaque = bundle_data;
 971     bundle_data->prefix = strdup(rsc->id);
 972 
 973     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_DOCKER, NULL,
 974                                    NULL);
 975     if (xml_obj != NULL) {
 976         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
 977     }
 978 
 979     if (xml_obj == NULL) {
 980         xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PODMAN, NULL,
 981                                        NULL);
 982         if (xml_obj != NULL) {
 983             bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
 984         }
 985     }
 986 
 987     if (xml_obj == NULL) {
 988         return FALSE;
 989     }
 990 
 991     // Use 0 for default, minimum, and invalid PCMK_XA_PROMOTED_MAX
 992     value = crm_element_value(xml_obj, PCMK_XA_PROMOTED_MAX);
 993     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
 994 
 995     /* Default replicas to PCMK_XA_PROMOTED_MAX if it was specified and 1
 996      * otherwise
 997      */
 998     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS);
 999     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1000         bundle_data->nreplicas = bundle_data->promoted_max;
1001     } else {
1002         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1003     }
1004 
1005     /*
1006      * Communication between containers on the same host via the
1007      * floating IPs only works if the container is started with:
1008      *   --userland-proxy=false --ip-masq=false
1009      */
1010     value = crm_element_value(xml_obj, PCMK_XA_REPLICAS_PER_HOST);
1011     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1012     if (bundle_data->nreplicas_per_host == 1) {
1013         pcmk__clear_rsc_flags(rsc, pcmk__rsc_unique);
1014     }
1015 
1016     bundle_data->container_command =
1017         crm_element_value_copy(xml_obj, PCMK_XA_RUN_COMMAND);
1018     bundle_data->launcher_options = crm_element_value_copy(xml_obj,
1019                                                            PCMK_XA_OPTIONS);
1020     bundle_data->image = crm_element_value_copy(xml_obj, PCMK_XA_IMAGE);
1021     bundle_data->container_network = crm_element_value_copy(xml_obj,
1022                                                             PCMK_XA_NETWORK);
1023 
1024     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_NETWORK, NULL,
1025                                    NULL);
1026     if(xml_obj) {
1027         bundle_data->ip_range_start =
1028             crm_element_value_copy(xml_obj, PCMK_XA_IP_RANGE_START);
1029         bundle_data->host_netmask =
1030             crm_element_value_copy(xml_obj, PCMK_XA_HOST_NETMASK);
1031         bundle_data->host_network =
1032             crm_element_value_copy(xml_obj, PCMK_XA_HOST_INTERFACE);
1033         bundle_data->control_port =
1034             crm_element_value_copy(xml_obj, PCMK_XA_CONTROL_PORT);
1035         value = crm_element_value(xml_obj, PCMK_XA_ADD_HOST);
1036         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1037             bundle_data->add_host = TRUE;
1038         }
1039 
1040         for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_PORT_MAPPING,
1041                                               NULL, NULL);
1042              xml_child != NULL;
1043              xml_child = pcmk__xe_next(xml_child, PCMK_XE_PORT_MAPPING)) {
1044 
1045             pe__bundle_port_t *port =
1046                 pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1047 
1048             port->source = crm_element_value_copy(xml_child, PCMK_XA_PORT);
1049 
1050             if(port->source == NULL) {
1051                 port->source = crm_element_value_copy(xml_child, PCMK_XA_RANGE);
1052             } else {
1053                 port->target = crm_element_value_copy(xml_child,
1054                                                       PCMK_XA_INTERNAL_PORT);
1055             }
1056 
1057             if(port->source != NULL && strlen(port->source) > 0) {
1058                 if(port->target == NULL) {
1059                     port->target = strdup(port->source);
1060                 }
1061                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1062 
1063             } else {
1064                 pcmk__config_err("Invalid " PCMK_XA_PORT " directive %s",
1065                                  pcmk__xe_id(xml_child));
1066                 port_free(port);
1067             }
1068         }
1069     }
1070 
1071     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_STORAGE, NULL,
1072                                    NULL);
1073     for (xml_child = pcmk__xe_first_child(xml_obj, PCMK_XE_STORAGE_MAPPING,
1074                                           NULL, NULL);
1075          xml_child != NULL;
1076          xml_child = pcmk__xe_next(xml_child, PCMK_XE_STORAGE_MAPPING)) {
1077 
1078         const char *source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR);
1079         const char *target = crm_element_value(xml_child, PCMK_XA_TARGET_DIR);
1080         const char *options = crm_element_value(xml_child, PCMK_XA_OPTIONS);
1081         int flags = pe__bundle_mount_none;
1082 
1083         if (source == NULL) {
1084             source = crm_element_value(xml_child, PCMK_XA_SOURCE_DIR_ROOT);
1085             pe__set_bundle_mount_flags(xml_child, flags,
1086                                        pe__bundle_mount_subdir);
1087         }
1088 
1089         if (source && target) {
1090             mount_add(bundle_data, source, target, options, flags);
1091             if (strcmp(target, "/var/log") == 0) {
1092                 need_log_mount = FALSE;
1093             }
1094         } else {
1095             pcmk__config_err("Invalid mount directive %s",
1096                              pcmk__xe_id(xml_child));
1097         }
1098     }
1099 
1100     xml_obj = pcmk__xe_first_child(rsc->priv->xml, PCMK_XE_PRIMITIVE, NULL,
1101                                    NULL);
1102     if (xml_obj && valid_network(bundle_data)) {
1103         const char *suffix = NULL;
1104         char *value = NULL;
1105         xmlNode *xml_set = NULL;
1106 
1107         xml_resource = pcmk__xe_create(NULL, PCMK_XE_CLONE);
1108 
1109         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1110          * part of the resource name, so that bundles don't restart in a rolling
1111          * upgrade. (It also avoids needing to change regression tests.)
1112          */
1113         suffix = (const char *) xml_resource->name;
1114         if (bundle_data->promoted_max > 0) {
1115             suffix = "master";
1116         }
1117 
1118         pcmk__xe_set_id(xml_resource, "%s-%s", bundle_data->prefix, suffix);
1119 
1120         xml_set = pcmk__xe_create(xml_resource, PCMK_XE_META_ATTRIBUTES);
1121         pcmk__xe_set_id(xml_set, "%s-%s-meta",
1122                         bundle_data->prefix, xml_resource->name);
1123 
1124         crm_create_nvpair_xml(xml_set, NULL,
1125                               PCMK_META_ORDERED, PCMK_VALUE_TRUE);
1126 
1127         value = pcmk__itoa(bundle_data->nreplicas);
1128         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_MAX, value);
1129         free(value);
1130 
1131         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1132         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_CLONE_NODE_MAX, value);
1133         free(value);
1134 
1135         crm_create_nvpair_xml(xml_set, NULL, PCMK_META_GLOBALLY_UNIQUE,
1136                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1137 
1138         if (bundle_data->promoted_max) {
1139             crm_create_nvpair_xml(xml_set, NULL,
1140                                   PCMK_META_PROMOTABLE, PCMK_VALUE_TRUE);
1141 
1142             value = pcmk__itoa(bundle_data->promoted_max);
1143             crm_create_nvpair_xml(xml_set, NULL, PCMK_META_PROMOTED_MAX, value);
1144             free(value);
1145         }
1146 
1147         //crm_xml_add(xml_obj, PCMK_XA_ID, bundle_data->prefix);
1148         pcmk__xml_copy(xml_resource, xml_obj);
1149 
1150     } else if(xml_obj) {
1151         pcmk__config_err("Cannot control %s inside %s without either "
1152                          PCMK_XA_IP_RANGE_START " or " PCMK_XA_CONTROL_PORT,
1153                          rsc->id, pcmk__xe_id(xml_obj));
1154         return FALSE;
1155     }
1156 
1157     if(xml_resource) {
1158         int lpc = 0;
1159         GList *childIter = NULL;
1160         pe__bundle_port_t *port = NULL;
1161         GString *buffer = NULL;
1162 
1163         if (pe__unpack_resource(xml_resource, &(bundle_data->child), rsc,
1164                                 rsc->priv->scheduler) != pcmk_rc_ok) {
1165             return FALSE;
1166         }
1167 
1168         /* Currently, we always map the default authentication key location
1169          * into the same location inside the container.
1170          *
1171          * Ideally, we would respect the host's PCMK_authkey_location, but:
1172          * - it may be different on different nodes;
1173          * - the actual connection will do extra checking to make sure the key
1174          *   file exists and is readable, that we can't do here on the DC
1175          * - tools such as crm_resource and crm_simulate may not have the same
1176          *   environment variables as the cluster, causing operation digests to
1177          *   differ
1178          *
1179          * Always using the default location inside the container is fine,
1180          * because we control the pacemaker_remote environment, and it avoids
1181          * having to pass another environment variable to the container.
1182          *
1183          * @TODO A better solution may be to have only pacemaker_remote use the
1184          * environment variable, and have the cluster nodes use a new
1185          * cluster option for key location. This would introduce the limitation
1186          * of the location being the same on all cluster nodes, but that's
1187          * reasonable.
1188          */
1189         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1190                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1191 
1192         if (need_log_mount) {
1193             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1194                       pe__bundle_mount_subdir);
1195         }
1196 
1197         port = pcmk__assert_alloc(1, sizeof(pe__bundle_port_t));
1198         if(bundle_data->control_port) {
1199             port->source = strdup(bundle_data->control_port);
1200         } else {
1201             /* If we wanted to respect PCMK_remote_port, we could use
1202              * crm_default_remote_port() here and elsewhere in this file instead
1203              * of DEFAULT_REMOTE_PORT.
1204              *
1205              * However, it gains nothing, since we control both the container
1206              * environment and the connection resource parameters, and the user
1207              * can use a different port if desired by setting
1208              * PCMK_XA_CONTROL_PORT.
1209              */
1210             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1211         }
1212         port->target = strdup(port->source);
1213         bundle_data->ports = g_list_append(bundle_data->ports, port);
1214 
1215         buffer = g_string_sized_new(1024);
1216         for (childIter = bundle_data->child->priv->children;
1217              childIter != NULL; childIter = childIter->next) {
1218 
1219             pcmk__bundle_replica_t *replica = NULL;
1220 
1221             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1222             replica->child = childIter->data;
1223             pcmk__set_rsc_flags(replica->child, pcmk__rsc_exclusive_probes);
1224             replica->offset = lpc++;
1225 
1226             // Ensure the child's notify gets set based on the underlying primitive's value
1227             if (pcmk_is_set(replica->child->flags, pcmk__rsc_notify)) {
1228                 pcmk__set_rsc_flags(bundle_data->child, pcmk__rsc_notify);
1229             }
1230 
1231             allocate_ip(bundle_data, replica, buffer);
1232             bundle_data->replicas = g_list_append(bundle_data->replicas,
1233                                                   replica);
1234             // coverity[null_field] replica->child can't be NULL here
1235             bundle_data->attribute_target =
1236                 g_hash_table_lookup(replica->child->priv->meta,
1237                                     PCMK_META_CONTAINER_ATTRIBUTE_TARGET);
1238         }
1239         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1240 
1241         if (bundle_data->attribute_target) {
1242             pcmk__insert_dup(rsc->priv->meta,
1243                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1244                              bundle_data->attribute_target);
1245             pcmk__insert_dup(bundle_data->child->priv->meta,
1246                              PCMK_META_CONTAINER_ATTRIBUTE_TARGET,
1247                              bundle_data->attribute_target);
1248         }
1249 
1250     } else {
1251         // Just a naked container, no pacemaker-remote
1252         GString *buffer = g_string_sized_new(1024);
1253 
1254         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1255             pcmk__bundle_replica_t *replica = NULL;
1256 
1257             replica = pcmk__assert_alloc(1, sizeof(pcmk__bundle_replica_t));
1258             replica->offset = lpc;
1259             allocate_ip(bundle_data, replica, buffer);
1260             bundle_data->replicas = g_list_append(bundle_data->replicas,
1261                                                   replica);
1262         }
1263         bundle_data->container_host_options = g_string_free(buffer, FALSE);
1264     }
1265 
1266     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1267          gIter = gIter->next) {
1268         pcmk__bundle_replica_t *replica = gIter->data;
1269 
1270         if (create_replica_resources(rsc, bundle_data, replica) != pcmk_rc_ok) {
1271             pcmk__config_err("Failed unpacking resource %s", rsc->id);
1272             pcmk__free_resource(rsc);
1273             return FALSE;
1274         }
1275 
1276         /* Utilization needs special handling for bundles. It makes no sense for
1277          * the inner primitive to have utilization, because it is tied
1278          * one-to-one to the guest node created by the container resource -- and
1279          * there's no way to set capacities for that guest node anyway.
1280          *
1281          * What the user really wants is to configure utilization for the
1282          * container. However, the schema only allows utilization for
1283          * primitives, and the container resource is implicit anyway, so the
1284          * user can *only* configure utilization for the inner primitive. If
1285          * they do, move the primitive's utilization values to the container.
1286          *
1287          * @TODO This means that bundles without an inner primitive can't have
1288          * utilization. An alternative might be to allow utilization values in
1289          * the top-level bundle XML in the schema, and copy those to each
1290          * container.
1291          */
1292         if (replica->child != NULL) {
1293             GHashTable *empty = replica->container->priv->utilization;
1294 
1295             replica->container->priv->utilization =
1296                 replica->child->priv->utilization;
1297 
1298             replica->child->priv->utilization = empty;
1299         }
1300     }
1301 
1302     if (bundle_data->child) {
1303         rsc->priv->children = g_list_append(rsc->priv->children,
1304                                             bundle_data->child);
1305     }
1306     return TRUE;
1307 }
1308 
1309 static int
1310 replica_resource_active(pcmk_resource_t *rsc, gboolean all)
     /*  */
1311 {
1312     if (rsc) {
1313         gboolean child_active = rsc->priv->fns->active(rsc, all);
1314 
1315         if (child_active && !all) {
1316             return TRUE;
1317         } else if (!child_active && all) {
1318             return FALSE;
1319         }
1320     }
1321     return -1;
1322 }
1323 
1324 bool
1325 pe__bundle_active(const pcmk_resource_t *rsc, bool all)
     /*  */
1326 {
1327     pe__bundle_variant_data_t *bundle_data = NULL;
1328     GList *iter = NULL;
1329 
1330     get_bundle_variant_data(bundle_data, rsc);
1331     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1332         pcmk__bundle_replica_t *replica = iter->data;
1333         int rsc_active;
1334 
1335         rsc_active = replica_resource_active(replica->ip, all);
1336         if (rsc_active >= 0) {
1337             return (bool) rsc_active;
1338         }
1339 
1340         rsc_active = replica_resource_active(replica->child, all);
1341         if (rsc_active >= 0) {
1342             return (bool) rsc_active;
1343         }
1344 
1345         rsc_active = replica_resource_active(replica->container, all);
1346         if (rsc_active >= 0) {
1347             return (bool) rsc_active;
1348         }
1349 
1350         rsc_active = replica_resource_active(replica->remote, all);
1351         if (rsc_active >= 0) {
1352             return (bool) rsc_active;
1353         }
1354     }
1355 
1356     /* If "all" is TRUE, we've already checked that no resources were inactive,
1357      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1358      * so return FALSE.
1359      */
1360     return all;
1361 }
1362 
1363 /*!
1364  * \internal
1365  * \brief Find the bundle replica corresponding to a given node
1366  *
1367  * \param[in] bundle  Top-level bundle resource
1368  * \param[in] node    Node to search for
1369  *
1370  * \return Bundle replica if found, NULL otherwise
1371  */
1372 pcmk_resource_t *
1373 pe__find_bundle_replica(const pcmk_resource_t *bundle, const pcmk_node_t *node)
     /*  */
1374 {
1375     pe__bundle_variant_data_t *bundle_data = NULL;
1376 
1377     pcmk__assert((bundle != NULL) && (node != NULL));
1378 
1379     get_bundle_variant_data(bundle_data, bundle);
1380     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1381          gIter = gIter->next) {
1382         pcmk__bundle_replica_t *replica = gIter->data;
1383 
1384         pcmk__assert((replica != NULL) && (replica->node != NULL));
1385         if (pcmk__same_node(replica->node, node)) {
1386             return replica->child;
1387         }
1388     }
1389     return NULL;
1390 }
1391 
1392 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1393                   "GList *")
1394 int
1395 pe__bundle_xml(pcmk__output_t *out, va_list args)
1396 {
1397     uint32_t show_opts = va_arg(args, uint32_t);
1398     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1399     GList *only_node = va_arg(args, GList *);
1400     GList *only_rsc = va_arg(args, GList *);
1401 
1402     pe__bundle_variant_data_t *bundle_data = NULL;
1403     int rc = pcmk_rc_no_output;
1404     gboolean printed_header = FALSE;
1405     bool print_everything = true;
1406 
1407     const char *desc = NULL;
1408 
1409     pcmk__assert(rsc != NULL);
1410     get_bundle_variant_data(bundle_data, rsc);
1411 
1412     if (rsc->priv->fns->is_filtered(rsc, only_rsc, true)) {
1413         return rc;
1414     }
1415 
1416     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1417 
1418     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1419          gIter = gIter->next) {
1420         pcmk__bundle_replica_t *replica = gIter->data;
1421         pcmk_resource_t *ip = replica->ip;
1422         pcmk_resource_t *child = replica->child;
1423         pcmk_resource_t *container = replica->container;
1424         pcmk_resource_t *remote = replica->remote;
1425         char *id = NULL;
1426         gboolean print_ip, print_child, print_ctnr, print_remote;
1427 
1428         pcmk__assert(replica != NULL);
1429 
1430         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1431             continue;
1432         }
1433 
1434         print_ip = (ip != NULL)
1435                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1436                                                   print_everything);
1437         print_child = (child != NULL)
1438                       && !child->priv->fns->is_filtered(child, only_rsc,
1439                                                         print_everything);
1440         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1441                                                         print_everything);
1442         print_remote = (remote != NULL)
1443                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1444                                                           print_everything);
1445 
1446         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1447             continue;
1448         }
1449 
1450         if (!printed_header) {
1451             const char *type = container_agent_str(bundle_data->agent_type);
1452             const char *unique = pcmk__flag_text(rsc->flags, pcmk__rsc_unique);
1453             const char *maintenance = pcmk__flag_text(rsc->flags,
1454                                                       pcmk__rsc_maintenance);
1455             const char *managed = pcmk__flag_text(rsc->flags,
1456                                                   pcmk__rsc_managed);
1457             const char *failed = pcmk__flag_text(rsc->flags, pcmk__rsc_failed);
1458 
1459             printed_header = TRUE;
1460 
1461             desc = pe__resource_description(rsc, show_opts);
1462 
1463             rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_BUNDLE,
1464                                           PCMK_XA_ID, rsc->id,
1465                                           PCMK_XA_TYPE, type,
1466                                           PCMK_XA_IMAGE, bundle_data->image,
1467                                           PCMK_XA_UNIQUE, unique,
1468                                           PCMK_XA_MAINTENANCE, maintenance,
1469                                           PCMK_XA_MANAGED, managed,
1470                                           PCMK_XA_FAILED, failed,
1471                                           PCMK_XA_DESCRIPTION, desc,
1472                                           NULL);
1473             pcmk__assert(rc == pcmk_rc_ok);
1474         }
1475 
1476         id = pcmk__itoa(replica->offset);
1477         rc = pe__name_and_nvpairs_xml(out, true, PCMK_XE_REPLICA,
1478                                       PCMK_XA_ID, id,
1479                                       NULL);
1480         free(id);
1481         pcmk__assert(rc == pcmk_rc_ok);
1482 
1483         if (print_ip) {
1484             out->message(out, (const char *) ip->priv->xml->name, show_opts,
1485                          ip, only_node, only_rsc);
1486         }
1487 
1488         if (print_child) {
1489             out->message(out, (const char *) child->priv->xml->name,
1490                          show_opts, child, only_node, only_rsc);
1491         }
1492 
1493         if (print_ctnr) {
1494             out->message(out, (const char *) container->priv->xml->name,
1495                          show_opts, container, only_node, only_rsc);
1496         }
1497 
1498         if (print_remote) {
1499             out->message(out, (const char *) remote->priv->xml->name,
1500                          show_opts, remote, only_node, only_rsc);
1501         }
1502 
1503         pcmk__output_xml_pop_parent(out); // replica
1504     }
1505 
1506     if (printed_header) {
1507         pcmk__output_xml_pop_parent(out); // bundle
1508     }
1509 
1510     return rc;
1511 }
1512 
1513 static void
1514 pe__bundle_replica_output_html(pcmk__output_t *out,
     /*  */
1515                                pcmk__bundle_replica_t *replica,
1516                                pcmk_node_t *node, uint32_t show_opts)
1517 {
1518     pcmk_resource_t *rsc = replica->child;
1519 
1520     int offset = 0;
1521     char buffer[LINE_MAX];
1522 
1523     if(rsc == NULL) {
1524         rsc = replica->container;
1525     }
1526 
1527     if (replica->remote) {
1528         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1529                            rsc_printable_id(replica->remote));
1530     } else {
1531         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1532                            rsc_printable_id(replica->container));
1533     }
1534     if (replica->ipaddr) {
1535         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1536                            replica->ipaddr);
1537     }
1538 
1539     pe__common_output_html(out, rsc, buffer, node, show_opts);
1540 }
1541 
1542 /*!
1543  * \internal
1544  * \brief Get a string describing a resource's unmanaged state or lack thereof
1545  *
1546  * \param[in] rsc  Resource to describe
1547  *
1548  * \return A string indicating that a resource is in maintenance mode or
1549  *         otherwise unmanaged, or an empty string otherwise
1550  */
1551 static const char *
1552 get_unmanaged_str(const pcmk_resource_t *rsc)
     /*  */
1553 {
1554     if (pcmk_is_set(rsc->flags, pcmk__rsc_maintenance)) {
1555         return " (maintenance)";
1556     }
1557     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
1558         return " (unmanaged)";
1559     }
1560     return "";
1561 }
1562 
1563 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1564                   "GList *")
1565 int
1566 pe__bundle_html(pcmk__output_t *out, va_list args)
1567 {
1568     uint32_t show_opts = va_arg(args, uint32_t);
1569     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1570     GList *only_node = va_arg(args, GList *);
1571     GList *only_rsc = va_arg(args, GList *);
1572 
1573     const char *desc = NULL;
1574     pe__bundle_variant_data_t *bundle_data = NULL;
1575     int rc = pcmk_rc_no_output;
1576     bool print_everything = true;
1577 
1578     pcmk__assert(rsc != NULL);
1579     get_bundle_variant_data(bundle_data, rsc);
1580 
1581     desc = pe__resource_description(rsc, show_opts);
1582 
1583     if (rsc->priv->fns->is_filtered(rsc, only_rsc, true)) {
1584         return rc;
1585     }
1586 
1587     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1588 
1589     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1590          gIter = gIter->next) {
1591         pcmk__bundle_replica_t *replica = gIter->data;
1592         pcmk_resource_t *ip = replica->ip;
1593         pcmk_resource_t *child = replica->child;
1594         pcmk_resource_t *container = replica->container;
1595         pcmk_resource_t *remote = replica->remote;
1596         gboolean print_ip, print_child, print_ctnr, print_remote;
1597 
1598         pcmk__assert(replica != NULL);
1599 
1600         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1601             continue;
1602         }
1603 
1604         print_ip = (ip != NULL)
1605                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1606                                                   print_everything);
1607         print_child = (child != NULL)
1608                       && !child->priv->fns->is_filtered(child, only_rsc,
1609                                                         print_everything);
1610         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1611                                                         print_everything);
1612         print_remote = (remote != NULL)
1613                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1614                                                           print_everything);
1615 
1616         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1617             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1618             /* The text output messages used below require pe_print_implicit to
1619              * be set to do anything.
1620              */
1621             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1622 
1623             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1624                                      (bundle_data->nreplicas > 1)? " set" : "",
1625                                      rsc->id, bundle_data->image,
1626                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1627                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1628                                      get_unmanaged_str(rsc));
1629 
1630             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1631                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1632             }
1633 
1634             if (print_ip) {
1635                 out->message(out, (const char *) ip->priv->xml->name,
1636                              new_show_opts, ip, only_node, only_rsc);
1637             }
1638 
1639             if (print_child) {
1640                 out->message(out, (const char *) child->priv->xml->name,
1641                              new_show_opts, child, only_node, only_rsc);
1642             }
1643 
1644             if (print_ctnr) {
1645                 out->message(out, (const char *) container->priv->xml->name,
1646                              new_show_opts, container, only_node, only_rsc);
1647             }
1648 
1649             if (print_remote) {
1650                 out->message(out, (const char *) remote->priv->xml->name,
1651                              new_show_opts, remote, only_node, only_rsc);
1652             }
1653 
1654             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1655                 out->end_list(out);
1656             }
1657         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1658             continue;
1659         } else {
1660             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1661                                      (bundle_data->nreplicas > 1)? " set" : "",
1662                                      rsc->id, bundle_data->image,
1663                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1664                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1665                                      get_unmanaged_str(rsc));
1666 
1667             pe__bundle_replica_output_html(out, replica,
1668                                            pcmk__current_node(container),
1669                                            show_opts);
1670         }
1671     }
1672 
1673     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1674     return rc;
1675 }
1676 
1677 static void
1678 pe__bundle_replica_output_text(pcmk__output_t *out,
     /*  */
1679                                pcmk__bundle_replica_t *replica,
1680                                pcmk_node_t *node, uint32_t show_opts)
1681 {
1682     const pcmk_resource_t *rsc = replica->child;
1683 
1684     int offset = 0;
1685     char buffer[LINE_MAX];
1686 
1687     if(rsc == NULL) {
1688         rsc = replica->container;
1689     }
1690 
1691     if (replica->remote) {
1692         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1693                            rsc_printable_id(replica->remote));
1694     } else {
1695         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1696                            rsc_printable_id(replica->container));
1697     }
1698     if (replica->ipaddr) {
1699         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1700                            replica->ipaddr);
1701     }
1702 
1703     pe__common_output_text(out, rsc, buffer, node, show_opts);
1704 }
1705 
1706 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pcmk_resource_t *", "GList *",
     /*  */
1707                   "GList *")
1708 int
1709 pe__bundle_text(pcmk__output_t *out, va_list args)
1710 {
1711     uint32_t show_opts = va_arg(args, uint32_t);
1712     pcmk_resource_t *rsc = va_arg(args, pcmk_resource_t *);
1713     GList *only_node = va_arg(args, GList *);
1714     GList *only_rsc = va_arg(args, GList *);
1715 
1716     const char *desc = NULL;
1717     pe__bundle_variant_data_t *bundle_data = NULL;
1718     int rc = pcmk_rc_no_output;
1719     bool print_everything = true;
1720 
1721     desc = pe__resource_description(rsc, show_opts);
1722 
1723     pcmk__assert(rsc != NULL);
1724     get_bundle_variant_data(bundle_data, rsc);
1725 
1726     if (rsc->priv->fns->is_filtered(rsc, only_rsc, true)) {
1727         return rc;
1728     }
1729 
1730     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1731 
1732     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1733          gIter = gIter->next) {
1734         pcmk__bundle_replica_t *replica = gIter->data;
1735         pcmk_resource_t *ip = replica->ip;
1736         pcmk_resource_t *child = replica->child;
1737         pcmk_resource_t *container = replica->container;
1738         pcmk_resource_t *remote = replica->remote;
1739         gboolean print_ip, print_child, print_ctnr, print_remote;
1740 
1741         pcmk__assert(replica != NULL);
1742 
1743         if (pcmk__rsc_filtered_by_node(container, only_node)) {
1744             continue;
1745         }
1746 
1747         print_ip = (ip != NULL)
1748                    && !ip->priv->fns->is_filtered(ip, only_rsc,
1749                                                   print_everything);
1750         print_child = (child != NULL)
1751                       && !child->priv->fns->is_filtered(child, only_rsc,
1752                                                         print_everything);
1753         print_ctnr = !container->priv->fns->is_filtered(container, only_rsc,
1754                                                         print_everything);
1755         print_remote = (remote != NULL)
1756                        && !remote->priv->fns->is_filtered(remote, only_rsc,
1757                                                           print_everything);
1758 
1759         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1760             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1761             /* The text output messages used below require pe_print_implicit to
1762              * be set to do anything.
1763              */
1764             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1765 
1766             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1767                                      (bundle_data->nreplicas > 1)? " set" : "",
1768                                      rsc->id, bundle_data->image,
1769                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1770                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1771                                      get_unmanaged_str(rsc));
1772 
1773             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1774                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1775             }
1776 
1777             out->begin_list(out, NULL, NULL, NULL);
1778 
1779             if (print_ip) {
1780                 out->message(out, (const char *) ip->priv->xml->name,
1781                              new_show_opts, ip, only_node, only_rsc);
1782             }
1783 
1784             if (print_child) {
1785                 out->message(out, (const char *) child->priv->xml->name,
1786                              new_show_opts, child, only_node, only_rsc);
1787             }
1788 
1789             if (print_ctnr) {
1790                 out->message(out, (const char *) container->priv->xml->name,
1791                              new_show_opts, container, only_node, only_rsc);
1792             }
1793 
1794             if (print_remote) {
1795                 out->message(out, (const char *) remote->priv->xml->name,
1796                              new_show_opts, remote, only_node, only_rsc);
1797             }
1798 
1799             out->end_list(out);
1800         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1801             continue;
1802         } else {
1803             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s%s%s%s",
1804                                      (bundle_data->nreplicas > 1)? " set" : "",
1805                                      rsc->id, bundle_data->image,
1806                                      pcmk_is_set(rsc->flags, pcmk__rsc_unique)? " (unique)" : "",
1807                                      desc ? " (" : "", desc ? desc : "", desc ? ")" : "",
1808                                      get_unmanaged_str(rsc));
1809 
1810             pe__bundle_replica_output_text(out, replica,
1811                                            pcmk__current_node(container),
1812                                            show_opts);
1813         }
1814     }
1815 
1816     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1817     return rc;
1818 }
1819 
1820 static void
1821 free_bundle_replica(pcmk__bundle_replica_t *replica)
     /*  */
1822 {
1823     if (replica == NULL) {
1824         return;
1825     }
1826 
1827     pcmk__free_node_copy(replica->node);
1828     replica->node = NULL;
1829 
1830     if (replica->ip) {
1831         pcmk__xml_free(replica->ip->priv->xml);
1832         replica->ip->priv->xml = NULL;
1833         pcmk__free_resource(replica->ip);
1834     }
1835     if (replica->container) {
1836         pcmk__xml_free(replica->container->priv->xml);
1837         replica->container->priv->xml = NULL;
1838         pcmk__free_resource(replica->container);
1839     }
1840     if (replica->remote) {
1841         pcmk__xml_free(replica->remote->priv->xml);
1842         replica->remote->priv->xml = NULL;
1843         pcmk__free_resource(replica->remote);
1844     }
1845     free(replica->ipaddr);
1846     free(replica);
1847 }
1848 
1849 void
1850 pe__free_bundle(pcmk_resource_t *rsc)
     /*  */
1851 {
1852     pe__bundle_variant_data_t *bundle_data = NULL;
1853     CRM_CHECK(rsc != NULL, return);
1854 
1855     get_bundle_variant_data(bundle_data, rsc);
1856     pcmk__rsc_trace(rsc, "Freeing %s", rsc->id);
1857 
1858     free(bundle_data->prefix);
1859     free(bundle_data->image);
1860     free(bundle_data->control_port);
1861     free(bundle_data->host_network);
1862     free(bundle_data->host_netmask);
1863     free(bundle_data->ip_range_start);
1864     free(bundle_data->container_network);
1865     free(bundle_data->launcher_options);
1866     free(bundle_data->container_command);
1867     g_free(bundle_data->container_host_options);
1868 
1869     g_list_free_full(bundle_data->replicas,
1870                      (GDestroyNotify) free_bundle_replica);
1871     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1872     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1873     g_list_free(rsc->priv->children);
1874 
1875     if(bundle_data->child) {
1876         pcmk__xml_free(bundle_data->child->priv->xml);
1877         bundle_data->child->priv->xml = NULL;
1878         pcmk__free_resource(bundle_data->child);
1879     }
1880     common_free(rsc);
1881 }
1882 
1883 enum rsc_role_e
1884 pe__bundle_resource_state(const pcmk_resource_t *rsc, bool current)
     /*  */
1885 {
1886     enum rsc_role_e container_role = pcmk_role_unknown;
1887     return container_role;
1888 }
1889 
1890 /*!
1891  * \brief Get the number of configured replicas in a bundle
1892  *
1893  * \param[in] rsc  Bundle resource
1894  *
1895  * \return Number of configured replicas, or 0 on error
1896  */
1897 int
1898 pe_bundle_replicas(const pcmk_resource_t *rsc)
     /*  */
1899 {
1900     if (pcmk__is_bundle(rsc)) {
1901         pe__bundle_variant_data_t *bundle_data = NULL;
1902 
1903         get_bundle_variant_data(bundle_data, rsc);
1904         return bundle_data->nreplicas;
1905     }
1906     return 0;
1907 }
1908 
1909 void
1910 pe__count_bundle(pcmk_resource_t *rsc)
     /*  */
1911 {
1912     pe__bundle_variant_data_t *bundle_data = NULL;
1913 
1914     get_bundle_variant_data(bundle_data, rsc);
1915     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
1916         pcmk__bundle_replica_t *replica = item->data;
1917 
1918         if (replica->ip) {
1919             replica->ip->priv->fns->count(replica->ip);
1920         }
1921         if (replica->child) {
1922             replica->child->priv->fns->count(replica->child);
1923         }
1924         if (replica->container) {
1925             replica->container->priv->fns->count(replica->container);
1926         }
1927         if (replica->remote) {
1928             replica->remote->priv->fns->count(replica->remote);
1929         }
1930     }
1931 }
1932 
1933 bool
1934 pe__bundle_is_filtered(const pcmk_resource_t *rsc, const GList *only_rsc,
     /*  */
1935                        bool check_parent)
1936 {
1937     bool passes = false;
1938     pe__bundle_variant_data_t *bundle_data = NULL;
1939 
1940     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
1941         passes = true;
1942     } else {
1943         get_bundle_variant_data(bundle_data, rsc);
1944 
1945         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
1946             pcmk__bundle_replica_t *replica = gIter->data;
1947             pcmk_resource_t *ip = replica->ip;
1948             pcmk_resource_t *child = replica->child;
1949             pcmk_resource_t *container = replica->container;
1950             pcmk_resource_t *remote = replica->remote;
1951 
1952             if ((ip != NULL)
1953                 && !ip->priv->fns->is_filtered(ip, only_rsc, false)) {
1954                 passes = true;
1955                 break;
1956             }
1957             if ((child != NULL)
1958                 && !child->priv->fns->is_filtered(child, only_rsc, false)) {
1959                 passes = true;
1960                 break;
1961             }
1962             if (!container->priv->fns->is_filtered(container, only_rsc,
1963                                                    false)) {
1964                 passes = true;
1965                 break;
1966             }
1967             if ((remote != NULL)
1968                 && !remote->priv->fns->is_filtered(remote, only_rsc, false)) {
1969                 passes = true;
1970                 break;
1971             }
1972         }
1973     }
1974 
1975     return !passes;
1976 }
1977 
1978 /*!
1979  * \internal
1980  * \brief Get a list of a bundle's containers
1981  *
1982  * \param[in] bundle  Bundle resource
1983  *
1984  * \return Newly created list of \p bundle's containers
1985  * \note It is the caller's responsibility to free the result with
1986  *       g_list_free().
1987  */
1988 GList *
1989 pe__bundle_containers(const pcmk_resource_t *bundle)
     /*  */
1990 {
1991     /* @TODO It would be more efficient to do this once when unpacking the
1992      * bundle, creating a new GList* in the variant data
1993      */
1994     GList *containers = NULL;
1995     const pe__bundle_variant_data_t *data = NULL;
1996 
1997     get_bundle_variant_data(data, bundle);
1998     for (GList *iter = data->replicas; iter != NULL; iter = iter->next) {
1999         pcmk__bundle_replica_t *replica = iter->data;
2000 
2001         containers = g_list_append(containers, replica->container);
2002     }
2003     return containers;
2004 }
2005 
2006 // Bundle implementation of pcmk__rsc_methods_t:active_node()
2007 pcmk_node_t *
2008 pe__bundle_active_node(const pcmk_resource_t *rsc, unsigned int *count_all,
     /*  */
2009                        unsigned int *count_clean)
2010 {
2011     pcmk_node_t *active = NULL;
2012     pcmk_node_t *node = NULL;
2013     pcmk_resource_t *container = NULL;
2014     GList *containers = NULL;
2015     GList *iter = NULL;
2016     GHashTable *nodes = NULL;
2017     const pe__bundle_variant_data_t *data = NULL;
2018 
2019     if (count_all != NULL) {
2020         *count_all = 0;
2021     }
2022     if (count_clean != NULL) {
2023         *count_clean = 0;
2024     }
2025     if (rsc == NULL) {
2026         return NULL;
2027     }
2028 
2029     /* For the purposes of this method, we only care about where the bundle's
2030      * containers are active, so build a list of active containers.
2031      */
2032     get_bundle_variant_data(data, rsc);
2033     for (iter = data->replicas; iter != NULL; iter = iter->next) {
2034         pcmk__bundle_replica_t *replica = iter->data;
2035 
2036         if (replica->container->priv->active_nodes != NULL) {
2037             containers = g_list_append(containers, replica->container);
2038         }
2039     }
2040     if (containers == NULL) {
2041         return NULL;
2042     }
2043 
2044     /* If the bundle has only a single active container, just use that
2045      * container's method. If live migration is ever supported for bundle
2046      * containers, this will allow us to prefer the migration source when there
2047      * is only one container and it is migrating. For now, this just lets us
2048      * avoid creating the nodes table.
2049      */
2050     if (pcmk__list_of_1(containers)) {
2051         container = containers->data;
2052         node = container->priv->fns->active_node(container, count_all,
2053                                                  count_clean);
2054         g_list_free(containers);
2055         return node;
2056     }
2057 
2058     // Add all containers' active nodes to a hash table (for uniqueness)
2059     nodes = g_hash_table_new(NULL, NULL);
2060     for (iter = containers; iter != NULL; iter = iter->next) {
2061         container = iter->data;
2062         for (GList *node_iter = container->priv->active_nodes;
2063              node_iter != NULL; node_iter = node_iter->next) {
2064 
2065             node = node_iter->data;
2066 
2067             // If insert returns true, we haven't counted this node yet
2068             if (g_hash_table_insert(nodes, (gpointer) node->details,
2069                                     (gpointer) node)
2070                 && !pe__count_active_node(rsc, node, &active, count_all,
2071                                           count_clean)) {
2072                 goto done;
2073             }
2074         }
2075     }
2076 
2077 done:
2078     g_list_free(containers);
2079     g_hash_table_destroy(nodes);
2080     return active;
2081 }
2082 
2083 /*!
2084  * \internal
2085  * \brief Get maximum bundle resource instances per node
2086  *
2087  * \param[in] rsc  Bundle resource to check
2088  *
2089  * \return Maximum number of \p rsc instances that can be active on one node
2090  */
2091 unsigned int
2092 pe__bundle_max_per_node(const pcmk_resource_t *rsc)
     /*  */
2093 {
2094     pe__bundle_variant_data_t *bundle_data = NULL;
2095 
2096     get_bundle_variant_data(bundle_data, rsc);
2097     pcmk__assert(bundle_data->nreplicas_per_host >= 0);
2098     return (unsigned int) bundle_data->nreplicas_per_host;
2099 }

/* */