root/lib/pengine/bundle.c

/* */

DEFINITIONS

1. next_ip
2. allocate_ip
3. create_resource
4. valid_network
5. create_ip_resource
6. create_docker_resource
7. create_podman_resource
8. create_rkt_resource
9. disallow_node
10. create_remote_resource
11. create_container
12. mount_add
13. mount_free
14. port_free
15. replica_for_remote
16. pe__bundle_needs_remote_name
17. pe__add_bundle_remote_name
18. pe__unpack_bundle
19. replica_resource_active
20. pe__bundle_active
21. pe__find_bundle_replica
22. print_rsc_in_list
23. container_agent_str
24. bundle_print_xml
25. PCMK__OUTPUT_ARGS
26. pe__bundle_replica_output_html
27. PCMK__OUTPUT_ARGS
28. pe__bundle_replica_output_text
29. PCMK__OUTPUT_ARGS
30. print_bundle_replica
31. pe__print_bundle
32. free_bundle_replica
33. pe__free_bundle
34. pe__bundle_resource_state
35. pe_bundle_replicas
36. pe__count_bundle
37. pe__bundle_is_filtered

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 
  14 #include <crm/pengine/rules.h>
  15 #include <crm/pengine/status.h>
  16 #include <crm/pengine/internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/output.h>
  19 #include <crm/common/xml_internal.h>
  20 #include <pe_status_private.h>
  21 
  22 #define PE__VARIANT_BUNDLE 1
  23 #include "./variant.h"
  24 
  25 static char *
  26 next_ip(const char *last_ip)
     /*  */
  27 {
  28     unsigned int oct1 = 0;
  29     unsigned int oct2 = 0;
  30     unsigned int oct3 = 0;
  31     unsigned int oct4 = 0;
  32     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  33 
  34     if (rc != 4) {
  35         /*@ TODO check for IPv6 */
  36         return NULL;
  37 
  38     } else if (oct3 > 253) {
  39         return NULL;
  40 
  41     } else if (oct4 > 253) {
  42         ++oct3;
  43         oct4 = 1;
  44 
  45     } else {
  46         ++oct4;
  47     }
  48 
  49     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  50 }
  51 
  52 static int
  53 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  54             char *buffer, int max)
  55 {
  56     if(data->ip_range_start == NULL) {
  57         return 0;
  58 
  59     } else if(data->ip_last) {
  60         replica->ipaddr = next_ip(data->ip_last);
  61 
  62     } else {
  63         replica->ipaddr = strdup(data->ip_range_start);
  64     }
  65 
  66     data->ip_last = replica->ipaddr;
  67     switch (data->agent_type) {
  68         case PE__CONTAINER_AGENT_DOCKER:
  69         case PE__CONTAINER_AGENT_PODMAN:
  70             if (data->add_host) {
  71                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
  72                                 data->prefix, replica->offset,
  73                                 replica->ipaddr);
  74             }
  75         case PE__CONTAINER_AGENT_RKT:
  76             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
  77                             replica->ipaddr, data->prefix, replica->offset);
  78         default: // PE__CONTAINER_AGENT_UNKNOWN
  79             break;
  80     }
  81     return 0;
  82 }
  83 
  84 static xmlNode *
  85 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
  86 {
  87     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
  88 
  89     crm_xml_add(rsc, XML_ATTR_ID, name);
  90     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
  91     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
  92     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
  93 
  94     return rsc;
  95 }
  96 
  97 /*!
  98  * \internal
  99  * \brief Check whether cluster can manage resource inside container
 100  *
 101  * \param[in] data  Container variant data
 102  *
 103  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 104  *
 105  * \note The resource is manageable if an IP range or control port has been
 106  *       specified. If a control port is used without an IP range, replicas per
 107  *       host must be 1.
 108  */
 109 static bool
 110 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 111 {
 112     if(data->ip_range_start) {
 113         return TRUE;
 114     }
 115     if(data->control_port) {
 116         if(data->nreplicas_per_host > 1) {
 117             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 118             data->nreplicas_per_host = 1;
 119             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 120         }
 121         return TRUE;
 122     }
 123     return FALSE;
 124 }
 125 
 126 static bool
 127 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 128                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 129 {
 130     if(data->ip_range_start) {
 131         char *id = NULL;
 132         xmlNode *xml_ip = NULL;
 133         xmlNode *xml_obj = NULL;
 134 
 135         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 136         crm_xml_sanitize_id(id);
 137         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 138         free(id);
 139 
 140         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 141         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 142                        data->prefix, replica->offset);
 143 
 144         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 145         if(data->host_network) {
 146             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 147         }
 148 
 149         if(data->host_netmask) {
 150             crm_create_nvpair_xml(xml_obj, NULL,
 151                                   "cidr_netmask", data->host_netmask);
 152 
 153         } else {
 154             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 155         }
 156 
 157         xml_obj = create_xml_node(xml_ip, "operations");
 158         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 159 
 160         // TODO: Other ops? Timeouts and intervals from underlying resource?
 161 
 162         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
 163             return FALSE;
 164         }
 165 
 166         parent->children = g_list_append(parent->children, replica->ip);
 167     }
 168     return TRUE;
 169 }
 170 
 171 static bool
 172 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 173                        pe__bundle_replica_t *replica,
 174                        pe_working_set_t *data_set)
 175 {
 176         int offset = 0, max = 4096;
 177         char *buffer = calloc(1, max+1);
 178 
 179         int doffset = 0, dmax = 1024;
 180         char *dbuffer = calloc(1, dmax+1);
 181 
 182         char *id = NULL;
 183         xmlNode *xml_container = NULL;
 184         xmlNode *xml_obj = NULL;
 185 
 186         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
 187         crm_xml_sanitize_id(id);
 188         xml_container = create_resource(id, "heartbeat",
 189                                         PE__CONTAINER_AGENT_DOCKER_S);
 190         free(id);
 191 
 192         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 193         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 194                        data->prefix, replica->offset);
 195 
 196         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 197         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 198         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 199         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 200 
 201         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 202 
 203         /* Set a container hostname only if we have an IP to map it to.
 204          * The user can set -h or --uts=host themselves if they want a nicer
 205          * name for logs, but this makes applications happy who need their
 206          * hostname to match the IP they bind to.
 207          */
 208         if (data->ip_range_start != NULL) {
 209             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 210                                data->prefix, replica->offset);
 211         }
 212 
 213         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 214 
 215         if (data->container_network) {
 216 #if 0
 217             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 218                                replica->ipaddr);
 219 #endif
 220             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 221                                data->container_network);
 222         }
 223 
 224         if(data->control_port) {
 225             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 226         } else {
 227             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 228         }
 229 
 230         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 231             pe__bundle_mount_t *mount = pIter->data;
 232 
 233             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 234                 char *source = crm_strdup_printf(
 235                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 236 
 237                 if(doffset > 0) {
 238                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 239                 }
 240                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 241                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 242                 free(source);
 243 
 244             } else {
 245                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 246             }
 247             if(mount->options) {
 248                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 249             }
 250         }
 251 
 252         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 253             pe__bundle_port_t *port = pIter->data;
 254 
 255             if (replica->ipaddr) {
 256                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 257                                    replica->ipaddr, port->source,
 258                                    port->target);
 259             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 260                 // No need to do port mapping if net=host
 261                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 262             }
 263         }
 264 
 265         if (data->launcher_options) {
 266             offset += snprintf(buffer+offset, max-offset, " %s",
 267                                data->launcher_options);
 268         }
 269 
 270         if (data->container_host_options) {
 271             offset += snprintf(buffer + offset, max - offset, " %s",
 272                                data->container_host_options);
 273         }
 274 
 275         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 276         free(buffer);
 277 
 278         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 279         free(dbuffer);
 280 
 281         if (replica->child) {
 282             if (data->container_command) {
 283                 crm_create_nvpair_xml(xml_obj, NULL,
 284                                       "run_cmd", data->container_command);
 285             } else {
 286                 crm_create_nvpair_xml(xml_obj, NULL,
 287                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 288             }
 289 
 290             /* TODO: Allow users to specify their own?
 291              *
 292              * We just want to know if the container is alive, we'll
 293              * monitor the child independently
 294              */
 295             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 296 #if 0
 297         /* @TODO Consider supporting the use case where we can start and stop
 298          * resources, but not proxy local commands (such as setting node
 299          * attributes), by running the local executor in stand-alone mode.
 300          * However, this would probably be better done via ACLs as with other
 301          * Pacemaker Remote nodes.
 302          */
 303         } else if ((child != NULL) && data->untrusted) {
 304             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 305                                   CRM_DAEMON_DIR "/pacemaker-execd");
 306             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 307                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 308 #endif
 309         } else {
 310             if (data->container_command) {
 311                 crm_create_nvpair_xml(xml_obj, NULL,
 312                                       "run_cmd", data->container_command);
 313             }
 314 
 315             /* TODO: Allow users to specify their own?
 316              *
 317              * We don't know what's in the container, so we just want
 318              * to know if it is alive
 319              */
 320             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 321         }
 322 
 323 
 324         xml_obj = create_xml_node(xml_container, "operations");
 325         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 326 
 327         // TODO: Other ops? Timeouts and intervals from underlying resource?
 328         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 329             return FALSE;
 330         }
 331         parent->children = g_list_append(parent->children, replica->container);
 332         return TRUE;
 333 }
 334 
 335 static bool
 336 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 337                        pe__bundle_replica_t *replica,
 338                        pe_working_set_t *data_set)
 339 {
 340         int offset = 0, max = 4096;
 341         char *buffer = calloc(1, max+1);
 342 
 343         int doffset = 0, dmax = 1024;
 344         char *dbuffer = calloc(1, dmax+1);
 345 
 346         char *id = NULL;
 347         xmlNode *xml_container = NULL;
 348         xmlNode *xml_obj = NULL;
 349 
 350         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
 351         crm_xml_sanitize_id(id);
 352         xml_container = create_resource(id, "heartbeat",
 353                                         PE__CONTAINER_AGENT_PODMAN_S);
 354         free(id);
 355 
 356         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 357         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 358                        data->prefix, replica->offset);
 359 
 360         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 361         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 362         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 363         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 364 
 365         // FIXME: (bandini 2018-08) podman has no restart policies
 366         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 367 
 368         /* Set a container hostname only if we have an IP to map it to.
 369          * The user can set -h or --uts=host themselves if they want a nicer
 370          * name for logs, but this makes applications happy who need their
 371          * hostname to match the IP they bind to.
 372          */
 373         if (data->ip_range_start != NULL) {
 374             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 375                                data->prefix, replica->offset);
 376         }
 377 
 378         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 379 
 380         if (data->container_network) {
 381 #if 0
 382             // podman has no support for --link-local-ip
 383             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 384                                replica->ipaddr);
 385 #endif
 386             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 387                                data->container_network);
 388         }
 389 
 390         if(data->control_port) {
 391             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 392         } else {
 393             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 394         }
 395 
 396         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 397             pe__bundle_mount_t *mount = pIter->data;
 398 
 399             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 400                 char *source = crm_strdup_printf(
 401                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 402 
 403                 if(doffset > 0) {
 404                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 405                 }
 406                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 407                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 408                 free(source);
 409 
 410             } else {
 411                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 412             }
 413             if(mount->options) {
 414                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 415             }
 416         }
 417 
 418         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 419             pe__bundle_port_t *port = pIter->data;
 420 
 421             if (replica->ipaddr) {
 422                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 423                                    replica->ipaddr, port->source,
 424                                    port->target);
 425             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 426                 // No need to do port mapping if net=host
 427                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 428             }
 429         }
 430 
 431         if (data->launcher_options) {
 432             offset += snprintf(buffer+offset, max-offset, " %s",
 433                                data->launcher_options);
 434         }
 435 
 436         if (data->container_host_options) {
 437             offset += snprintf(buffer + offset, max - offset, " %s",
 438                                data->container_host_options);
 439         }
 440 
 441         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 442         free(buffer);
 443 
 444         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 445         free(dbuffer);
 446 
 447         if (replica->child) {
 448             if (data->container_command) {
 449                 crm_create_nvpair_xml(xml_obj, NULL,
 450                                       "run_cmd", data->container_command);
 451             } else {
 452                 crm_create_nvpair_xml(xml_obj, NULL,
 453                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 454             }
 455 
 456             /* TODO: Allow users to specify their own?
 457              *
 458              * We just want to know if the container is alive, we'll
 459              * monitor the child independently
 460              */
 461             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 462 #if 0
 463         /* @TODO Consider supporting the use case where we can start and stop
 464          * resources, but not proxy local commands (such as setting node
 465          * attributes), by running the local executor in stand-alone mode.
 466          * However, this would probably be better done via ACLs as with other
 467          * Pacemaker Remote nodes.
 468          */
 469         } else if ((child != NULL) && data->untrusted) {
 470             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 471                                   CRM_DAEMON_DIR "/pacemaker-execd");
 472             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 473                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 474 #endif
 475         } else {
 476             if (data->container_command) {
 477                 crm_create_nvpair_xml(xml_obj, NULL,
 478                                       "run_cmd", data->container_command);
 479             }
 480 
 481             /* TODO: Allow users to specify their own?
 482              *
 483              * We don't know what's in the container, so we just want
 484              * to know if it is alive
 485              */
 486             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 487         }
 488 
 489 
 490         xml_obj = create_xml_node(xml_container, "operations");
 491         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 492 
 493         // TODO: Other ops? Timeouts and intervals from underlying resource?
 494         if (!common_unpack(xml_container, &replica->container, parent,
 495                            data_set)) {
 496             return FALSE;
 497         }
 498         parent->children = g_list_append(parent->children, replica->container);
 499         return TRUE;
 500 }
 501 
 502 static bool
 503 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 504                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 505 {
 506         int offset = 0, max = 4096;
 507         char *buffer = calloc(1, max+1);
 508 
 509         int doffset = 0, dmax = 1024;
 510         char *dbuffer = calloc(1, dmax+1);
 511 
 512         char *id = NULL;
 513         xmlNode *xml_container = NULL;
 514         xmlNode *xml_obj = NULL;
 515 
 516         int volid = 0;
 517 
 518         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
 519         crm_xml_sanitize_id(id);
 520         xml_container = create_resource(id, "heartbeat",
 521                                         PE__CONTAINER_AGENT_RKT_S);
 522         free(id);
 523 
 524         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 525         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 526                        data->prefix, replica->offset);
 527 
 528         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 529         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
 530         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
 531         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 532 
 533         /* Set a container hostname only if we have an IP to map it to.
 534          * The user can set -h or --uts=host themselves if they want a nicer
 535          * name for logs, but this makes applications happy who need their
 536          * hostname to match the IP they bind to.
 537          */
 538         if (data->ip_range_start != NULL) {
 539             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
 540                                data->prefix, replica->offset);
 541         }
 542 
 543         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 544 
 545         if (data->container_network) {
 546 #if 0
 547             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 548                                replica->ipaddr);
 549 #endif
 550             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 551                                data->container_network);
 552         }
 553 
 554         if(data->control_port) {
 555             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
 556         } else {
 557             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 558         }
 559 
 560         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 561             pe__bundle_mount_t *mount = pIter->data;
 562 
 563             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 564                 char *source = crm_strdup_printf(
 565                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 566 
 567                 if(doffset > 0) {
 568                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 569                 }
 570                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 571                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
 572                 if(mount->options) {
 573                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 574                 }
 575                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 576                 free(source);
 577 
 578             } else {
 579                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
 580                 if(mount->options) {
 581                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 582                 }
 583                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 584             }
 585             volid++;
 586         }
 587 
 588         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 589             pe__bundle_port_t *port = pIter->data;
 590 
 591             if (replica->ipaddr) {
 592                 offset += snprintf(buffer+offset, max-offset,
 593                                    " --port=%s:%s:%s", port->target,
 594                                    replica->ipaddr, port->source);
 595             } else {
 596                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
 597             }
 598         }
 599 
 600         if (data->launcher_options) {
 601             offset += snprintf(buffer+offset, max-offset, " %s",
 602                                data->launcher_options);
 603         }
 604 
 605         if (data->container_host_options) {
 606             offset += snprintf(buffer + offset, max - offset, " %s",
 607                                data->container_host_options);
 608         }
 609 
 610         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 611         free(buffer);
 612 
 613         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 614         free(dbuffer);
 615 
 616         if (replica->child) {
 617             if (data->container_command) {
 618                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 619                                       data->container_command);
 620             } else {
 621                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 622                                       SBIN_DIR "/pacemaker-remoted");
 623             }
 624 
 625             /* TODO: Allow users to specify their own?
 626              *
 627              * We just want to know if the container is alive, we'll
 628              * monitor the child independently
 629              */
 630             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 631 #if 0
 632         /* @TODO Consider supporting the use case where we can start and stop
 633          * resources, but not proxy local commands (such as setting node
 634          * attributes), by running the local executor in stand-alone mode.
 635          * However, this would probably be better done via ACLs as with other
 636          * Pacemaker Remote nodes.
 637          */
 638         } else if ((child != NULL) && data->untrusted) {
 639             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 640                                   CRM_DAEMON_DIR "/pacemaker-execd");
 641             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 642                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 643 #endif
 644         } else {
 645             if (data->container_command) {
 646                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 647                                       data->container_command);
 648             }
 649 
 650             /* TODO: Allow users to specify their own?
 651              *
 652              * We don't know what's in the container, so we just want
 653              * to know if it is alive
 654              */
 655             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 656         }
 657 
 658 
 659         xml_obj = create_xml_node(xml_container, "operations");
 660         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 661 
 662         // TODO: Other ops? Timeouts and intervals from underlying resource?
 663 
 664         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 665             return FALSE;
 666         }
 667         parent->children = g_list_append(parent->children, replica->container);
 668         return TRUE;
 669 }
 670 
 671 /*!
 672  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 673  *
 674  * \param[in,out] rsc    Resource to modify
 675  * \param[in]     uname  Name of node to ban
 676  */
 677 static void
 678 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 679 {
 680     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 681 
 682     if (match) {
 683         ((pe_node_t *) match)->weight = -INFINITY;
 684         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 685     }
 686     if (rsc->children) {
 687         GList *child;
 688 
 689         for (child = rsc->children; child != NULL; child = child->next) {
 690             disallow_node((pe_resource_t *) (child->data), uname);
 691         }
 692     }
 693 }
 694 
 695 static bool
 696 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 697                        pe__bundle_replica_t *replica,
 698                        pe_working_set_t *data_set)
 699 {
 700     if (replica->child && valid_network(data)) {
 701         GHashTableIter gIter;
 702         GList *rsc_iter = NULL;
 703         pe_node_t *node = NULL;
 704         xmlNode *xml_remote = NULL;
 705         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 706         char *port_s = NULL;
 707         const char *uname = NULL;
 708         const char *connect_name = NULL;
 709 
 710         if (pe_find_resource(data_set->resources, id) != NULL) {
 711             free(id);
 712             // The biggest hammer we have
 713             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 714                                    replica->child->id, replica->offset);
 715             //@TODO return false instead of asserting?
 716             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
 717         }
 718 
 719         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 720          * connection does not have its own IP is a magic string that we use to
 721          * support nested remotes (i.e. a bundle running on a remote node).
 722          */
 723         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 724 
 725         if (data->control_port == NULL) {
 726             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 727         }
 728 
 729         /* This sets replica->container as replica->remote's container, which is
 730          * similar to what happens with guest nodes. This is how the scheduler
 731          * knows that the bundle node is fenced by recovering the container, and
 732          * that remote should be ordered relative to the container.
 733          */
 734         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 735                                           NULL, NULL, NULL,
 736                                           connect_name, (data->control_port?
 737                                           data->control_port : port_s));
 738         free(port_s);
 739 
 740         /* Abandon our created ID, and pull the copy from the XML, because we
 741          * need something that will get freed during data set cleanup to use as
 742          * the node ID and uname.
 743          */
 744         free(id);
 745         id = NULL;
 746         uname = ID(xml_remote);
 747 
 748         /* Ensure a node has been created for the guest (it may have already
 749          * been, if it has a permanent node attribute), and ensure its weight is
 750          * -INFINITY so no other resources can run on it.
 751          */
 752         node = pe_find_node(data_set->nodes, uname);
 753         if (node == NULL) {
 754             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 755                                   data_set);
 756         } else {
 757             node->weight = -INFINITY;
 758         }
 759         node->rsc_discover_mode = pe_discover_never;
 760 
 761         /* unpack_remote_nodes() ensures that each remote node and guest node
 762          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 763          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 764          * what nodes will be needed, so we do it just above.
 765          *
 766          * Worse, that means that the node may have been utilized while
 767          * unpacking other resources, without our weight correction. The most
 768          * likely place for this to happen is when common_unpack() calls
 769          * resource_location() to set a default score in symmetric clusters.
 770          * This adds a node *copy* to each resource's allowed nodes, and these
 771          * copies will have the wrong weight.
 772          *
 773          * As a hacky workaround, fix those copies here.
 774          *
 775          * @TODO Possible alternative: ensure bundles are unpacked before other
 776          * resources, so the weight is correct before any copies are made.
 777          */
 778         for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
 779             disallow_node((pe_resource_t *) (rsc_iter->data), uname);
 780         }
 781 
 782         replica->node = pe__copy_node(node);
 783         replica->node->weight = 500;
 784         replica->node->rsc_discover_mode = pe_discover_exclusive;
 785 
 786         /* Ensure the node shows up as allowed and with the correct discovery set */
 787         if (replica->child->allowed_nodes != NULL) {
 788             g_hash_table_destroy(replica->child->allowed_nodes);
 789         }
 790         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 791         g_hash_table_insert(replica->child->allowed_nodes,
 792                             (gpointer) replica->node->details->id,
 793                             pe__copy_node(replica->node));
 794 
 795         {
 796             pe_node_t *copy = pe__copy_node(replica->node);
 797             copy->weight = -INFINITY;
 798             g_hash_table_insert(replica->child->parent->allowed_nodes,
 799                                 (gpointer) replica->node->details->id, copy);
 800         }
 801         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
 802             return FALSE;
 803         }
 804 
 805         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 806         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 807             if (pe__is_guest_or_remote_node(node)) {
 808                 /* Remote resources can only run on 'normal' cluster node */
 809                 node->weight = -INFINITY;
 810             }
 811         }
 812 
 813         replica->node->details->remote_rsc = replica->remote;
 814 
 815         // Ensure pe__is_guest_node() functions correctly immediately
 816         replica->remote->container = replica->container;
 817 
 818         /* A bundle's #kind is closer to "container" (guest node) than the
 819          * "remote" set by pe_create_node().
 820          */
 821         g_hash_table_insert(replica->node->details->attrs,
 822                             strdup(CRM_ATTR_KIND), strdup("container"));
 823 
 824         /* One effect of this is that setup_container() will add
 825          * replica->remote to replica->container's fillers, which will make
 826          * pe__resource_contains_guest_node() true for replica->container.
 827          *
 828          * replica->child does NOT get added to replica->container's fillers.
 829          * The only noticeable effect if it did would be for its fail count to
 830          * be taken into account when checking replica->container's migration
 831          * threshold.
 832          */
 833         parent->children = g_list_append(parent->children, replica->remote);
 834     }
 835     return TRUE;
 836 }
 837 
 838 static bool
 839 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 840                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 841 {
 842 
 843     switch (data->agent_type) {
 844         case PE__CONTAINER_AGENT_DOCKER:
 845             if (!create_docker_resource(parent, data, replica, data_set)) {
 846                 return FALSE;
 847             }
 848             break;
 849 
 850         case PE__CONTAINER_AGENT_PODMAN:
 851             if (!create_podman_resource(parent, data, replica, data_set)) {
 852                 return FALSE;
 853             }
 854             break;
 855 
 856         case PE__CONTAINER_AGENT_RKT:
 857             if (!create_rkt_resource(parent, data, replica, data_set)) {
 858                 return FALSE;
 859             }
 860             break;
 861         default: // PE__CONTAINER_AGENT_UNKNOWN
 862             return FALSE;
 863     }
 864 
 865     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
 866         return FALSE;
 867     }
 868     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
 869         return FALSE;
 870     }
 871     if (replica->child && replica->ipaddr) {
 872         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 873     }
 874 
 875     if (replica->remote) {
 876         /*
 877          * Allow the remote connection resource to be allocated to a
 878          * different node than the one on which the container is active.
 879          *
 880          * This makes it possible to have Pacemaker Remote nodes running
 881          * containers with pacemaker-remoted inside in order to start
 882          * services inside those containers.
 883          */
 884         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 885     }
 886 
 887     return TRUE;
 888 }
 889 
 890 static void
 891 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 892           const char *target, const char *options, uint32_t flags)
 893 {
 894     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 895 
 896     mount->source = strdup(source);
 897     mount->target = strdup(target);
 898     if (options) {
 899         mount->options = strdup(options);
 900     }
 901     mount->flags = flags;
 902     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 903 }
 904 
 905 static void
 906 mount_free(pe__bundle_mount_t *mount)
     /*  */
 907 {
 908     free(mount->source);
 909     free(mount->target);
 910     free(mount->options);
 911     free(mount);
 912 }
 913 
 914 static void
 915 port_free(pe__bundle_port_t *port)
     /*  */
 916 {
 917     free(port->source);
 918     free(port->target);
 919     free(port);
 920 }
 921 
 922 static pe__bundle_replica_t *
 923 replica_for_remote(pe_resource_t *remote)
     /*  */
 924 {
 925     pe_resource_t *top = remote;
 926     pe__bundle_variant_data_t *bundle_data = NULL;
 927 
 928     if (top == NULL) {
 929         return NULL;
 930     }
 931 
 932     while (top->parent != NULL) {
 933         top = top->parent;
 934     }
 935 
 936     get_bundle_variant_data(bundle_data, top);
 937     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 938          gIter = gIter->next) {
 939         pe__bundle_replica_t *replica = gIter->data;
 940 
 941         if (replica->remote == remote) {
 942             return replica;
 943         }
 944     }
 945     CRM_LOG_ASSERT(FALSE);
 946     return NULL;
 947 }
 948 
 949 bool
 950 pe__bundle_needs_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 951 {
 952     const char *value;
 953     GHashTable *params = NULL;
 954 
 955     if (rsc == NULL) {
 956         return false;
 957     }
 958 
 959     // Use NULL node since pcmk__bundle_expand() uses that to set value
 960     params = pe_rsc_params(rsc, NULL, data_set);
 961     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 962 
 963     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 964            && xml_contains_remote_node(rsc->xml);
 965 }
 966 
 967 const char *
 968 pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
     /*  */
 969                            xmlNode *xml, const char *field)
 970 {
 971     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 972 
 973     pe_node_t *node = NULL;
 974     pe__bundle_replica_t *replica = NULL;
 975 
 976     if (!pe__bundle_needs_remote_name(rsc, data_set)) {
 977         return NULL;
 978     }
 979 
 980     replica = replica_for_remote(rsc);
 981     if (replica == NULL) {
 982         return NULL;
 983     }
 984 
 985     node = replica->container->allocated_to;
 986     if (node == NULL) {
 987         /* If it won't be running anywhere after the
 988          * transition, go with where it's running now.
 989          */
 990         node = pe__current_node(replica->container);
 991     }
 992 
 993     if(node == NULL) {
 994         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 995         return NULL;
 996     }
 997 
 998     crm_trace("Setting address for bundle connection %s to bundle host %s",
 999               rsc->id, node->details->uname);
1000     if(xml != NULL && field != NULL) {
1001         crm_xml_add(xml, field, node->details->uname);
1002     }
1003 
1004     return node->details->uname;
1005 }
1006 
1007 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
1008         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
1009                                    "Bundle mount", ID(mount_xml), flags,    \
1010                                    (flags_to_set), #flags_to_set);          \
1011     } while (0)
1012 
1013 gboolean
1014 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
1015 {
1016     const char *value = NULL;
1017     xmlNode *xml_obj = NULL;
1018     xmlNode *xml_resource = NULL;
1019     pe__bundle_variant_data_t *bundle_data = NULL;
1020     bool need_log_mount = TRUE;
1021 
1022     CRM_ASSERT(rsc != NULL);
1023     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
1024 
1025     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
1026     rsc->variant_opaque = bundle_data;
1027     bundle_data->prefix = strdup(rsc->id);
1028 
1029     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1030     if (xml_obj != NULL) {
1031         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1032     } else {
1033         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1034         if (xml_obj != NULL) {
1035             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1036         } else {
1037             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1038             if (xml_obj != NULL) {
1039                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1040             } else {
1041                 return FALSE;
1042             }
1043         }
1044     }
1045 
1046     // Use 0 for default, minimum, and invalid promoted-max
1047     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
1048     if (value == NULL) {
1049         // @COMPAT deprecated since 2.0.0
1050         value = crm_element_value(xml_obj, "masters");
1051     }
1052     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1053 
1054     // Default replicas to promoted-max if it was specified and 1 otherwise
1055     value = crm_element_value(xml_obj, "replicas");
1056     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1057         bundle_data->nreplicas = bundle_data->promoted_max;
1058     } else {
1059         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1060     }
1061 
1062     /*
1063      * Communication between containers on the same host via the
1064      * floating IPs only works if the container is started with:
1065      *   --userland-proxy=false --ip-masq=false
1066      */
1067     value = crm_element_value(xml_obj, "replicas-per-host");
1068     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1069     if (bundle_data->nreplicas_per_host == 1) {
1070         pe__clear_resource_flags(rsc, pe_rsc_unique);
1071     }
1072 
1073     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1074     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1075     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1076     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1077 
1078     xml_obj = first_named_child(rsc->xml, "network");
1079     if(xml_obj) {
1080 
1081         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1082         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1083         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1084         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1085         value = crm_element_value(xml_obj, "add-host");
1086         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1087             bundle_data->add_host = TRUE;
1088         }
1089 
1090         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1091              xml_child = pcmk__xe_next(xml_child)) {
1092 
1093             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1094             port->source = crm_element_value_copy(xml_child, "port");
1095 
1096             if(port->source == NULL) {
1097                 port->source = crm_element_value_copy(xml_child, "range");
1098             } else {
1099                 port->target = crm_element_value_copy(xml_child, "internal-port");
1100             }
1101 
1102             if(port->source != NULL && strlen(port->source) > 0) {
1103                 if(port->target == NULL) {
1104                     port->target = strdup(port->source);
1105                 }
1106                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1107 
1108             } else {
1109                 pe_err("Invalid port directive %s", ID(xml_child));
1110                 port_free(port);
1111             }
1112         }
1113     }
1114 
1115     xml_obj = first_named_child(rsc->xml, "storage");
1116     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1117          xml_child = pcmk__xe_next(xml_child)) {
1118 
1119         const char *source = crm_element_value(xml_child, "source-dir");
1120         const char *target = crm_element_value(xml_child, "target-dir");
1121         const char *options = crm_element_value(xml_child, "options");
1122         int flags = pe__bundle_mount_none;
1123 
1124         if (source == NULL) {
1125             source = crm_element_value(xml_child, "source-dir-root");
1126             pe__set_bundle_mount_flags(xml_child, flags,
1127                                        pe__bundle_mount_subdir);
1128         }
1129 
1130         if (source && target) {
1131             mount_add(bundle_data, source, target, options, flags);
1132             if (strcmp(target, "/var/log") == 0) {
1133                 need_log_mount = FALSE;
1134             }
1135         } else {
1136             pe_err("Invalid mount directive %s", ID(xml_child));
1137         }
1138     }
1139 
1140     xml_obj = first_named_child(rsc->xml, "primitive");
1141     if (xml_obj && valid_network(bundle_data)) {
1142         char *value = NULL;
1143         xmlNode *xml_set = NULL;
1144 
1145         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1146 
1147         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1148          * part of the resource name, so that bundles don't restart in a rolling
1149          * upgrade. (It also avoids needing to change regression tests.)
1150          */
1151         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1152                       (bundle_data->promoted_max? "master"
1153                       : (const char *)xml_resource->name));
1154 
1155         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1156         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1157 
1158         crm_create_nvpair_xml(xml_set, NULL,
1159                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1160 
1161         value = pcmk__itoa(bundle_data->nreplicas);
1162         crm_create_nvpair_xml(xml_set, NULL,
1163                               XML_RSC_ATTR_INCARNATION_MAX, value);
1164         free(value);
1165 
1166         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1167         crm_create_nvpair_xml(xml_set, NULL,
1168                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
1169         free(value);
1170 
1171         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1172                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1173 
1174         if (bundle_data->promoted_max) {
1175             crm_create_nvpair_xml(xml_set, NULL,
1176                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1177 
1178             value = pcmk__itoa(bundle_data->promoted_max);
1179             crm_create_nvpair_xml(xml_set, NULL,
1180                                   XML_RSC_ATTR_PROMOTED_MAX, value);
1181             free(value);
1182         }
1183 
1184         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1185         add_node_copy(xml_resource, xml_obj);
1186 
1187     } else if(xml_obj) {
1188         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1189                rsc->id, ID(xml_obj));
1190         return FALSE;
1191     }
1192 
1193     if(xml_resource) {
1194         int lpc = 0;
1195         GList *childIter = NULL;
1196         pe_resource_t *new_rsc = NULL;
1197         pe__bundle_port_t *port = NULL;
1198 
1199         int offset = 0, max = 1024;
1200         char *buffer = NULL;
1201 
1202         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
1203             pe_err("Failed unpacking resource %s", ID(rsc->xml));
1204             if (new_rsc != NULL && new_rsc->fns != NULL) {
1205                 new_rsc->fns->free(new_rsc);
1206             }
1207             return FALSE;
1208         }
1209 
1210         bundle_data->child = new_rsc;
1211 
1212         /* Currently, we always map the default authentication key location
1213          * into the same location inside the container.
1214          *
1215          * Ideally, we would respect the host's PCMK_authkey_location, but:
1216          * - it may be different on different nodes;
1217          * - the actual connection will do extra checking to make sure the key
1218          *   file exists and is readable, that we can't do here on the DC
1219          * - tools such as crm_resource and crm_simulate may not have the same
1220          *   environment variables as the cluster, causing operation digests to
1221          *   differ
1222          *
1223          * Always using the default location inside the container is fine,
1224          * because we control the pacemaker_remote environment, and it avoids
1225          * having to pass another environment variable to the container.
1226          *
1227          * @TODO A better solution may be to have only pacemaker_remote use the
1228          * environment variable, and have the cluster nodes use a new
1229          * cluster option for key location. This would introduce the limitation
1230          * of the location being the same on all cluster nodes, but that's
1231          * reasonable.
1232          */
1233         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1234                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1235 
1236         if (need_log_mount) {
1237             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1238                       pe__bundle_mount_subdir);
1239         }
1240 
1241         port = calloc(1, sizeof(pe__bundle_port_t));
1242         if(bundle_data->control_port) {
1243             port->source = strdup(bundle_data->control_port);
1244         } else {
1245             /* If we wanted to respect PCMK_remote_port, we could use
1246              * crm_default_remote_port() here and elsewhere in this file instead
1247              * of DEFAULT_REMOTE_PORT.
1248              *
1249              * However, it gains nothing, since we control both the container
1250              * environment and the connection resource parameters, and the user
1251              * can use a different port if desired by setting control-port.
1252              */
1253             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1254         }
1255         port->target = strdup(port->source);
1256         bundle_data->ports = g_list_append(bundle_data->ports, port);
1257 
1258         buffer = calloc(1, max+1);
1259         for (childIter = bundle_data->child->children; childIter != NULL;
1260              childIter = childIter->next) {
1261 
1262             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1263 
1264             replica->child = childIter->data;
1265             replica->child->exclusive_discover = TRUE;
1266             replica->offset = lpc++;
1267 
1268             // Ensure the child's notify gets set based on the underlying primitive's value
1269             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1270                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1271             }
1272 
1273             offset += allocate_ip(bundle_data, replica, buffer+offset,
1274                                   max-offset);
1275             bundle_data->replicas = g_list_append(bundle_data->replicas,
1276                                                   replica);
1277             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1278                                                                 XML_RSC_ATTR_TARGET);
1279         }
1280         bundle_data->container_host_options = buffer;
1281         if (bundle_data->attribute_target) {
1282             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1283                                  strdup(bundle_data->attribute_target));
1284             g_hash_table_replace(bundle_data->child->meta,
1285                                  strdup(XML_RSC_ATTR_TARGET),
1286                                  strdup(bundle_data->attribute_target));
1287         }
1288 
1289     } else {
1290         // Just a naked container, no pacemaker-remote
1291         int offset = 0, max = 1024;
1292         char *buffer = calloc(1, max+1);
1293 
1294         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1295             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1296 
1297             replica->offset = lpc;
1298             offset += allocate_ip(bundle_data, replica, buffer+offset,
1299                                   max-offset);
1300             bundle_data->replicas = g_list_append(bundle_data->replicas,
1301                                                   replica);
1302         }
1303         bundle_data->container_host_options = buffer;
1304     }
1305 
1306     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1307          gIter = gIter->next) {
1308         pe__bundle_replica_t *replica = gIter->data;
1309 
1310         if (!create_container(rsc, bundle_data, replica, data_set)) {
1311             pe_err("Failed unpacking resource %s", rsc->id);
1312             rsc->fns->free(rsc);
1313             return FALSE;
1314         }
1315     }
1316 
1317     if (bundle_data->child) {
1318         rsc->children = g_list_append(rsc->children, bundle_data->child);
1319     }
1320     return TRUE;
1321 }
1322 
1323 static int
1324 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1325 {
1326     if (rsc) {
1327         gboolean child_active = rsc->fns->active(rsc, all);
1328 
1329         if (child_active && !all) {
1330             return TRUE;
1331         } else if (!child_active && all) {
1332             return FALSE;
1333         }
1334     }
1335     return -1;
1336 }
1337 
1338 gboolean
1339 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1340 {
1341     pe__bundle_variant_data_t *bundle_data = NULL;
1342     GList *iter = NULL;
1343 
1344     get_bundle_variant_data(bundle_data, rsc);
1345     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1346         pe__bundle_replica_t *replica = iter->data;
1347         int rsc_active;
1348 
1349         rsc_active = replica_resource_active(replica->ip, all);
1350         if (rsc_active >= 0) {
1351             return (gboolean) rsc_active;
1352         }
1353 
1354         rsc_active = replica_resource_active(replica->child, all);
1355         if (rsc_active >= 0) {
1356             return (gboolean) rsc_active;
1357         }
1358 
1359         rsc_active = replica_resource_active(replica->container, all);
1360         if (rsc_active >= 0) {
1361             return (gboolean) rsc_active;
1362         }
1363 
1364         rsc_active = replica_resource_active(replica->remote, all);
1365         if (rsc_active >= 0) {
1366             return (gboolean) rsc_active;
1367         }
1368     }
1369 
1370     /* If "all" is TRUE, we've already checked that no resources were inactive,
1371      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1372      * so return FALSE.
1373      */
1374     return all;
1375 }
1376 
1377 /*!
1378  * \internal
1379  * \brief Find the bundle replica corresponding to a given node
1380  *
1381  * \param[in] bundle  Top-level bundle resource
1382  * \param[in] node    Node to search for
1383  *
1384  * \return Bundle replica if found, NULL otherwise
1385  */
1386 pe_resource_t *
1387 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1388 {
1389     pe__bundle_variant_data_t *bundle_data = NULL;
1390     CRM_ASSERT(bundle && node);
1391 
1392     get_bundle_variant_data(bundle_data, bundle);
1393     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1394          gIter = gIter->next) {
1395         pe__bundle_replica_t *replica = gIter->data;
1396 
1397         CRM_ASSERT(replica && replica->node);
1398         if (replica->node->details == node->details) {
1399             return replica->child;
1400         }
1401     }
1402     return NULL;
1403 }
1404 
1405 static void
1406 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1407                   void *print_data)
1408 {
1409     if (rsc != NULL) {
1410         if (options & pe_print_html) {
1411             status_print("<li>");
1412         }
1413         rsc->fns->print(rsc, pre_text, options, print_data);
1414         if (options & pe_print_html) {
1415             status_print("</li>\n");
1416         }
1417     }
1418 }
1419 
1420 static const char*
1421 container_agent_str(enum pe__container_agent t)
     /*  */
1422 {
1423     switch (t) {
1424         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
1425         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
1426         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
1427         default: // PE__CONTAINER_AGENT_UNKNOWN
1428             break;
1429     }
1430     return PE__CONTAINER_AGENT_UNKNOWN_S;
1431 }
1432 
1433 static void
1434 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1435                  void *print_data)
1436 {
1437     pe__bundle_variant_data_t *bundle_data = NULL;
1438     char *child_text = NULL;
1439     CRM_CHECK(rsc != NULL, return);
1440 
1441     if (pre_text == NULL) {
1442         pre_text = "";
1443     }
1444     child_text = crm_strdup_printf("%s        ", pre_text);
1445 
1446     get_bundle_variant_data(bundle_data, rsc);
1447 
1448     status_print("%s<bundle ", pre_text);
1449     status_print("id=\"%s\" ", rsc->id);
1450     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1451     status_print("image=\"%s\" ", bundle_data->image);
1452     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1453     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1454     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1455     status_print(">\n");
1456 
1457     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1458          gIter = gIter->next) {
1459         pe__bundle_replica_t *replica = gIter->data;
1460 
1461         CRM_ASSERT(replica);
1462         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
1463         print_rsc_in_list(replica->ip, child_text, options, print_data);
1464         print_rsc_in_list(replica->child, child_text, options, print_data);
1465         print_rsc_in_list(replica->container, child_text, options, print_data);
1466         print_rsc_in_list(replica->remote, child_text, options, print_data);
1467         status_print("%s    </replica>\n", pre_text);
1468     }
1469     status_print("%s</bundle>\n", pre_text);
1470     free(child_text);
1471 }
1472 
1473 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1474 int
1475 pe__bundle_xml(pcmk__output_t *out, va_list args)
1476 {
1477     unsigned int show_opts = va_arg(args, unsigned int);
1478     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1479     GList *only_node = va_arg(args, GList *);
1480     GList *only_rsc = va_arg(args, GList *);
1481 
1482     pe__bundle_variant_data_t *bundle_data = NULL;
1483     int rc = pcmk_rc_no_output;
1484     gboolean printed_header = FALSE;
1485     gboolean print_everything = TRUE;
1486 
1487     CRM_ASSERT(rsc != NULL);
1488 
1489     get_bundle_variant_data(bundle_data, rsc);
1490 
1491     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1492         return rc;
1493     }
1494 
1495     print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
1496 
1497     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1498          gIter = gIter->next) {
1499         pe__bundle_replica_t *replica = gIter->data;
1500         char *id = NULL;
1501         gboolean print_ip, print_child, print_ctnr, print_remote;
1502 
1503         CRM_ASSERT(replica);
1504 
1505         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1506             continue;
1507         }
1508 
1509         print_ip = replica->ip != NULL &&
1510                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1511         print_child = replica->child != NULL &&
1512                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1513         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1514         print_remote = replica->remote != NULL &&
1515                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1516 
1517         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1518             continue;
1519         }
1520 
1521         if (!printed_header) {
1522             printed_header = TRUE;
1523 
1524             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6,
1525                      "id", rsc->id,
1526                      "type", container_agent_str(bundle_data->agent_type),
1527                      "image", bundle_data->image,
1528                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1529                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1530                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed));
1531             CRM_ASSERT(rc == pcmk_rc_ok);
1532         }
1533 
1534         id = pcmk__itoa(replica->offset);
1535         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1536         free(id);
1537         CRM_ASSERT(rc == pcmk_rc_ok);
1538 
1539         if (print_ip) {
1540             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
1541                          replica->ip, only_node, only_rsc);
1542         }
1543 
1544         if (print_child) {
1545             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
1546                          replica->child, only_node, only_rsc);
1547         }
1548 
1549         if (print_ctnr) {
1550             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
1551                          replica->container, only_node, only_rsc);
1552         }
1553 
1554         if (print_remote) {
1555             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
1556                          replica->remote, only_node, only_rsc);
1557         }
1558 
1559         pcmk__output_xml_pop_parent(out); // replica
1560     }
1561 
1562     if (printed_header) {
1563         pcmk__output_xml_pop_parent(out); // bundle
1564     }
1565 
1566     return rc;
1567 }
1568 
1569 static void
1570 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1571                                pe_node_t *node, unsigned int show_opts)
1572 {
1573     pe_resource_t *rsc = replica->child;
1574 
1575     int offset = 0;
1576     char buffer[LINE_MAX];
1577 
1578     if(rsc == NULL) {
1579         rsc = replica->container;
1580     }
1581 
1582     if (replica->remote) {
1583         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1584                            rsc_printable_id(replica->remote));
1585     } else {
1586         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1587                            rsc_printable_id(replica->container));
1588     }
1589     if (replica->ipaddr) {
1590         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1591                            replica->ipaddr);
1592     }
1593 
1594     pe__common_output_html(out, rsc, buffer, node, show_opts);
1595 }
1596 
1597 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1598 int
1599 pe__bundle_html(pcmk__output_t *out, va_list args)
1600 {
1601     unsigned int show_opts = va_arg(args, unsigned int);
1602     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1603     GList *only_node = va_arg(args, GList *);
1604     GList *only_rsc = va_arg(args, GList *);
1605 
1606     pe__bundle_variant_data_t *bundle_data = NULL;
1607     int rc = pcmk_rc_no_output;
1608     gboolean print_everything = TRUE;
1609 
1610     CRM_ASSERT(rsc != NULL);
1611 
1612     get_bundle_variant_data(bundle_data, rsc);
1613 
1614     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1615         return rc;
1616     }
1617 
1618     print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
1619 
1620     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1621          gIter = gIter->next) {
1622         pe__bundle_replica_t *replica = gIter->data;
1623         gboolean print_ip, print_child, print_ctnr, print_remote;
1624 
1625         CRM_ASSERT(replica);
1626 
1627         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1628             continue;
1629         }
1630 
1631         print_ip = replica->ip != NULL &&
1632                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1633         print_child = replica->child != NULL &&
1634                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1635         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1636         print_remote = replica->remote != NULL &&
1637                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1638 
1639         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1640             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1641             /* The text output messages used below require pe_print_implicit to
1642              * be set to do anything.
1643              */
1644             unsigned int new_show_opts = show_opts | pcmk_show_implicit_rscs;
1645 
1646             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1647                                      (bundle_data->nreplicas > 1)? " set" : "",
1648                                      rsc->id, bundle_data->image,
1649                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1650                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1651 
1652             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1653                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1654             }
1655 
1656             if (print_ip) {
1657                 out->message(out, crm_map_element_name(replica->ip->xml),
1658                              new_show_opts, replica->ip, only_node, only_rsc);
1659             }
1660 
1661             if (print_child) {
1662                 out->message(out, crm_map_element_name(replica->child->xml),
1663                              new_show_opts, replica->child, only_node, only_rsc);
1664             }
1665 
1666             if (print_ctnr) {
1667                 out->message(out, crm_map_element_name(replica->container->xml),
1668                              new_show_opts, replica->container, only_node, only_rsc);
1669             }
1670 
1671             if (print_remote) {
1672                 out->message(out, crm_map_element_name(replica->remote->xml),
1673                              new_show_opts, replica->remote, only_node, only_rsc);
1674             }
1675 
1676             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1677                 out->end_list(out);
1678             }
1679         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1680             continue;
1681         } else {
1682             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1683                                      (bundle_data->nreplicas > 1)? " set" : "",
1684                                      rsc->id, bundle_data->image,
1685                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1686                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1687 
1688             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1689                                            show_opts);
1690         }
1691     }
1692 
1693     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1694     return rc;
1695 }
1696 
1697 static void
1698 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1699                                pe_node_t *node, unsigned int show_opts)
1700 {
1701     pe_resource_t *rsc = replica->child;
1702 
1703     int offset = 0;
1704     char buffer[LINE_MAX];
1705 
1706     if(rsc == NULL) {
1707         rsc = replica->container;
1708     }
1709 
1710     if (replica->remote) {
1711         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1712                            rsc_printable_id(replica->remote));
1713     } else {
1714         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1715                            rsc_printable_id(replica->container));
1716     }
1717     if (replica->ipaddr) {
1718         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1719                            replica->ipaddr);
1720     }
1721 
1722     pe__common_output_text(out, rsc, buffer, node, show_opts);
1723 }
1724 
1725 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1726 int
1727 pe__bundle_text(pcmk__output_t *out, va_list args)
1728 {
1729     unsigned int show_opts = va_arg(args, unsigned int);
1730     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1731     GList *only_node = va_arg(args, GList *);
1732     GList *only_rsc = va_arg(args, GList *);
1733 
1734     pe__bundle_variant_data_t *bundle_data = NULL;
1735     int rc = pcmk_rc_no_output;
1736     gboolean print_everything = TRUE;
1737 
1738     get_bundle_variant_data(bundle_data, rsc);
1739 
1740     CRM_ASSERT(rsc != NULL);
1741 
1742     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1743         return rc;
1744     }
1745 
1746     print_everything = pcmk__str_in_list(only_rsc, rsc->id, pcmk__str_none);
1747 
1748     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1749          gIter = gIter->next) {
1750         pe__bundle_replica_t *replica = gIter->data;
1751         gboolean print_ip, print_child, print_ctnr, print_remote;
1752 
1753         CRM_ASSERT(replica);
1754 
1755         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1756             continue;
1757         }
1758 
1759         print_ip = replica->ip != NULL &&
1760                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1761         print_child = replica->child != NULL &&
1762                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1763         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1764         print_remote = replica->remote != NULL &&
1765                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1766 
1767         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1768             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1769             /* The text output messages used below require pe_print_implicit to
1770              * be set to do anything.
1771              */
1772             unsigned int new_show_opts = show_opts | pcmk_show_implicit_rscs;
1773 
1774             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1775                                      (bundle_data->nreplicas > 1)? " set" : "",
1776                                      rsc->id, bundle_data->image,
1777                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1778                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1779 
1780             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1781                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1782             }
1783 
1784             out->begin_list(out, NULL, NULL, NULL);
1785 
1786             if (print_ip) {
1787                 out->message(out, crm_map_element_name(replica->ip->xml),
1788                              new_show_opts, replica->ip, only_node, only_rsc);
1789             }
1790 
1791             if (print_child) {
1792                 out->message(out, crm_map_element_name(replica->child->xml),
1793                              new_show_opts, replica->child, only_node, only_rsc);
1794             }
1795 
1796             if (print_ctnr) {
1797                 out->message(out, crm_map_element_name(replica->container->xml),
1798                              new_show_opts, replica->container, only_node, only_rsc);
1799             }
1800 
1801             if (print_remote) {
1802                 out->message(out, crm_map_element_name(replica->remote->xml),
1803                              new_show_opts, replica->remote, only_node, only_rsc);
1804             }
1805 
1806             out->end_list(out);
1807         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1808             continue;
1809         } else {
1810             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1811                                      (bundle_data->nreplicas > 1)? " set" : "",
1812                                      rsc->id, bundle_data->image,
1813                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1814                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1815 
1816             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1817                                            show_opts);
1818         }
1819     }
1820 
1821     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1822     return rc;
1823 }
1824 
1825 static void
1826 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1827                      long options, void *print_data)
1828 {
1829     pe_node_t *node = NULL;
1830     pe_resource_t *rsc = replica->child;
1831 
1832     int offset = 0;
1833     char buffer[LINE_MAX];
1834 
1835     if(rsc == NULL) {
1836         rsc = replica->container;
1837     }
1838 
1839     if (replica->remote) {
1840         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1841                            rsc_printable_id(replica->remote));
1842     } else {
1843         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1844                            rsc_printable_id(replica->container));
1845     }
1846     if (replica->ipaddr) {
1847         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1848                            replica->ipaddr);
1849     }
1850 
1851     node = pe__current_node(replica->container);
1852     common_print(rsc, pre_text, buffer, node, options, print_data);
1853 }
1854 
1855 void
1856 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1857                  void *print_data)
1858 {
1859     pe__bundle_variant_data_t *bundle_data = NULL;
1860     char *child_text = NULL;
1861     CRM_CHECK(rsc != NULL, return);
1862 
1863     if (options & pe_print_xml) {
1864         bundle_print_xml(rsc, pre_text, options, print_data);
1865         return;
1866     }
1867 
1868     get_bundle_variant_data(bundle_data, rsc);
1869 
1870     if (pre_text == NULL) {
1871         pre_text = " ";
1872     }
1873 
1874     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1875                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1876                  rsc->id, bundle_data->image,
1877                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1878                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1879     if (options & pe_print_html) {
1880         status_print("<br />\n<ul>\n");
1881     }
1882 
1883 
1884     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1885          gIter = gIter->next) {
1886         pe__bundle_replica_t *replica = gIter->data;
1887 
1888         CRM_ASSERT(replica);
1889         if (options & pe_print_html) {
1890             status_print("<li>");
1891         }
1892 
1893         if (pcmk_is_set(options, pe_print_implicit)) {
1894             child_text = crm_strdup_printf("     %s", pre_text);
1895             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1896                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1897             }
1898             if (options & pe_print_html) {
1899                 status_print("<br />\n<ul>\n");
1900             }
1901             print_rsc_in_list(replica->ip, child_text, options, print_data);
1902             print_rsc_in_list(replica->container, child_text, options, print_data);
1903             print_rsc_in_list(replica->remote, child_text, options, print_data);
1904             print_rsc_in_list(replica->child, child_text, options, print_data);
1905             if (options & pe_print_html) {
1906                 status_print("</ul>\n");
1907             }
1908         } else {
1909             child_text = crm_strdup_printf("%s  ", pre_text);
1910             print_bundle_replica(replica, child_text, options, print_data);
1911         }
1912         free(child_text);
1913 
1914         if (options & pe_print_html) {
1915             status_print("</li>\n");
1916         }
1917     }
1918     if (options & pe_print_html) {
1919         status_print("</ul>\n");
1920     }
1921 }
1922 
1923 static void
1924 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1925 {
1926     if (replica == NULL) {
1927         return;
1928     }
1929 
1930     if (replica->node) {
1931         free(replica->node);
1932         replica->node = NULL;
1933     }
1934 
1935     if (replica->ip) {
1936         free_xml(replica->ip->xml);
1937         replica->ip->xml = NULL;
1938         replica->ip->fns->free(replica->ip);
1939         replica->ip = NULL;
1940     }
1941     if (replica->container) {
1942         free_xml(replica->container->xml);
1943         replica->container->xml = NULL;
1944         replica->container->fns->free(replica->container);
1945         replica->container = NULL;
1946     }
1947     if (replica->remote) {
1948         free_xml(replica->remote->xml);
1949         replica->remote->xml = NULL;
1950         replica->remote->fns->free(replica->remote);
1951         replica->remote = NULL;
1952     }
1953     free(replica->ipaddr);
1954     free(replica);
1955 }
1956 
1957 void
1958 pe__free_bundle(pe_resource_t *rsc)
     /*  */
1959 {
1960     pe__bundle_variant_data_t *bundle_data = NULL;
1961     CRM_CHECK(rsc != NULL, return);
1962 
1963     get_bundle_variant_data(bundle_data, rsc);
1964     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1965 
1966     free(bundle_data->prefix);
1967     free(bundle_data->image);
1968     free(bundle_data->control_port);
1969     free(bundle_data->host_network);
1970     free(bundle_data->host_netmask);
1971     free(bundle_data->ip_range_start);
1972     free(bundle_data->container_network);
1973     free(bundle_data->launcher_options);
1974     free(bundle_data->container_command);
1975     free(bundle_data->container_host_options);
1976 
1977     g_list_free_full(bundle_data->replicas,
1978                      (GDestroyNotify) free_bundle_replica);
1979     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1980     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1981     g_list_free(rsc->children);
1982 
1983     if(bundle_data->child) {
1984         free_xml(bundle_data->child->xml);
1985         bundle_data->child->xml = NULL;
1986         bundle_data->child->fns->free(bundle_data->child);
1987     }
1988     common_free(rsc);
1989 }
1990 
1991 enum rsc_role_e
1992 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
1993 {
1994     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
1995     return container_role;
1996 }
1997 
1998 /*!
1999  * \brief Get the number of configured replicas in a bundle
2000  *
2001  * \param[in] rsc  Bundle resource
2002  *
2003  * \return Number of configured replicas, or 0 on error
2004  */
2005 int
2006 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
2007 {
2008     if ((rsc == NULL) || (rsc->variant != pe_container)) {
2009         return 0;
2010     } else {
2011         pe__bundle_variant_data_t *bundle_data = NULL;
2012 
2013         get_bundle_variant_data(bundle_data, rsc);
2014         return bundle_data->nreplicas;
2015     }
2016 }
2017 
2018 void
2019 pe__count_bundle(pe_resource_t *rsc)
     /*  */
2020 {
2021     pe__bundle_variant_data_t *bundle_data = NULL;
2022 
2023     get_bundle_variant_data(bundle_data, rsc);
2024     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2025         pe__bundle_replica_t *replica = item->data;
2026 
2027         if (replica->ip) {
2028             replica->ip->fns->count(replica->ip);
2029         }
2030         if (replica->child) {
2031             replica->child->fns->count(replica->child);
2032         }
2033         if (replica->container) {
2034             replica->container->fns->count(replica->container);
2035         }
2036         if (replica->remote) {
2037             replica->remote->fns->count(replica->remote);
2038         }
2039     }
2040 }
2041 
2042 gboolean
2043 pe__bundle_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent)
     /*  */
2044 {
2045     gboolean passes = FALSE;
2046     pe__bundle_variant_data_t *bundle_data = NULL;
2047 
2048     if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc), pcmk__str_none)) {
2049         passes = TRUE;
2050     } else {
2051         get_bundle_variant_data(bundle_data, rsc);
2052 
2053         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2054             pe__bundle_replica_t *replica = gIter->data;
2055 
2056             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2057                 passes = TRUE;
2058                 break;
2059             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2060                 passes = TRUE;
2061                 break;
2062             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2063                 passes = TRUE;
2064                 break;
2065             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2066                 passes = TRUE;
2067                 break;
2068             }
2069         }
2070     }
2071 
2072     return !passes;
2073 }