lib/pengine/bundle.c

/* */
This source file includes following definitions.
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
create_docker_resource
create_podman_resource
create_rkt_resource
disallow_node
create_remote_resource
create_container
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
container_agent_str
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 
  14 #include <crm/pengine/rules.h>
  15 #include <crm/pengine/status.h>
  16 #include <crm/pengine/internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/output.h>
  19 #include <crm/common/xml_internal.h>
  20 #include <pe_status_private.h>
  21 
  22 #define PE__VARIANT_BUNDLE 1
  23 #include "./variant.h"
  24 
  25 static char *
  26 next_ip(const char *last_ip)
     /*  */
  27 {
  28     unsigned int oct1 = 0;
  29     unsigned int oct2 = 0;
  30     unsigned int oct3 = 0;
  31     unsigned int oct4 = 0;
  32     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  33 
  34     if (rc != 4) {
  35         /*@ TODO check for IPv6 */
  36         return NULL;
  37 
  38     } else if (oct3 > 253) {
  39         return NULL;
  40 
  41     } else if (oct4 > 253) {
  42         ++oct3;
  43         oct4 = 1;
  44 
  45     } else {
  46         ++oct4;
  47     }
  48 
  49     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  50 }
  51 
  52 static int
  53 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  54             char *buffer, int max)
  55 {
  56     if(data->ip_range_start == NULL) {
  57         return 0;
  58 
  59     } else if(data->ip_last) {
  60         replica->ipaddr = next_ip(data->ip_last);
  61 
  62     } else {
  63         replica->ipaddr = strdup(data->ip_range_start);
  64     }
  65 
  66     data->ip_last = replica->ipaddr;
  67     switch (data->agent_type) {
  68         case PE__CONTAINER_AGENT_DOCKER:
  69         case PE__CONTAINER_AGENT_PODMAN:
  70             if (data->add_host) {
  71                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
  72                                 data->prefix, replica->offset,
  73                                 replica->ipaddr);
  74             }
  75         case PE__CONTAINER_AGENT_RKT:
  76             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
  77                             replica->ipaddr, data->prefix, replica->offset);
  78         default: // PE__CONTAINER_AGENT_UNKNOWN
  79             break;
  80     }
  81     return 0;
  82 }
  83 
  84 static xmlNode *
  85 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
  86 {
  87     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
  88 
  89     crm_xml_add(rsc, XML_ATTR_ID, name);
  90     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
  91     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
  92     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
  93 
  94     return rsc;
  95 }
  96 
  97 /*!
  98  * \internal
  99  * \brief Check whether cluster can manage resource inside container
 100  *
 101  * \param[in] data  Container variant data
 102  *
 103  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 104  *
 105  * \note The resource is manageable if an IP range or control port has been
 106  *       specified. If a control port is used without an IP range, replicas per
 107  *       host must be 1.
 108  */
 109 static bool
 110 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 111 {
 112     if(data->ip_range_start) {
 113         return TRUE;
 114     }
 115     if(data->control_port) {
 116         if(data->nreplicas_per_host > 1) {
 117             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 118             data->nreplicas_per_host = 1;
 119             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 120         }
 121         return TRUE;
 122     }
 123     return FALSE;
 124 }
 125 
 126 static bool
 127 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 128                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 129 {
 130     if(data->ip_range_start) {
 131         char *id = NULL;
 132         xmlNode *xml_ip = NULL;
 133         xmlNode *xml_obj = NULL;
 134 
 135         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 136         crm_xml_sanitize_id(id);
 137         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 138         free(id);
 139 
 140         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 141         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 142                        data->prefix, replica->offset);
 143 
 144         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 145         if(data->host_network) {
 146             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 147         }
 148 
 149         if(data->host_netmask) {
 150             crm_create_nvpair_xml(xml_obj, NULL,
 151                                   "cidr_netmask", data->host_netmask);
 152 
 153         } else {
 154             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 155         }
 156 
 157         xml_obj = create_xml_node(xml_ip, "operations");
 158         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 159 
 160         // TODO: Other ops? Timeouts and intervals from underlying resource?
 161 
 162         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
 163             return FALSE;
 164         }
 165 
 166         parent->children = g_list_append(parent->children, replica->ip);
 167     }
 168     return TRUE;
 169 }
 170 
 171 static bool
 172 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 173                        pe__bundle_replica_t *replica,
 174                        pe_working_set_t *data_set)
 175 {
 176         int offset = 0, max = 4096;
 177         char *buffer = calloc(1, max+1);
 178 
 179         int doffset = 0, dmax = 1024;
 180         char *dbuffer = calloc(1, dmax+1);
 181 
 182         char *id = NULL;
 183         xmlNode *xml_container = NULL;
 184         xmlNode *xml_obj = NULL;
 185 
 186         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
 187         crm_xml_sanitize_id(id);
 188         xml_container = create_resource(id, "heartbeat",
 189                                         PE__CONTAINER_AGENT_DOCKER_S);
 190         free(id);
 191 
 192         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 193         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 194                        data->prefix, replica->offset);
 195 
 196         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 197         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 198         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 199         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 200 
 201         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 202 
 203         /* Set a container hostname only if we have an IP to map it to.
 204          * The user can set -h or --uts=host themselves if they want a nicer
 205          * name for logs, but this makes applications happy who need their
 206          * hostname to match the IP they bind to.
 207          */
 208         if (data->ip_range_start != NULL) {
 209             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 210                                data->prefix, replica->offset);
 211         }
 212 
 213         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 214 
 215         if (data->container_network) {
 216 #if 0
 217             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 218                                replica->ipaddr);
 219 #endif
 220             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 221                                data->container_network);
 222         }
 223 
 224         if(data->control_port) {
 225             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 226         } else {
 227             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 228         }
 229 
 230         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 231             pe__bundle_mount_t *mount = pIter->data;
 232 
 233             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 234                 char *source = crm_strdup_printf(
 235                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 236 
 237                 if(doffset > 0) {
 238                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 239                 }
 240                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 241                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 242                 free(source);
 243 
 244             } else {
 245                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 246             }
 247             if(mount->options) {
 248                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 249             }
 250         }
 251 
 252         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 253             pe__bundle_port_t *port = pIter->data;
 254 
 255             if (replica->ipaddr) {
 256                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 257                                    replica->ipaddr, port->source,
 258                                    port->target);
 259             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 260                 // No need to do port mapping if net=host
 261                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 262             }
 263         }
 264 
 265         if (data->launcher_options) {
 266             offset += snprintf(buffer+offset, max-offset, " %s",
 267                                data->launcher_options);
 268         }
 269 
 270         if (data->container_host_options) {
 271             offset += snprintf(buffer + offset, max - offset, " %s",
 272                                data->container_host_options);
 273         }
 274 
 275         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 276         free(buffer);
 277 
 278         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 279         free(dbuffer);
 280 
 281         if (replica->child) {
 282             if (data->container_command) {
 283                 crm_create_nvpair_xml(xml_obj, NULL,
 284                                       "run_cmd", data->container_command);
 285             } else {
 286                 crm_create_nvpair_xml(xml_obj, NULL,
 287                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 288             }
 289 
 290             /* TODO: Allow users to specify their own?
 291              *
 292              * We just want to know if the container is alive, we'll
 293              * monitor the child independently
 294              */
 295             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 296 #if 0
 297         /* @TODO Consider supporting the use case where we can start and stop
 298          * resources, but not proxy local commands (such as setting node
 299          * attributes), by running the local executor in stand-alone mode.
 300          * However, this would probably be better done via ACLs as with other
 301          * Pacemaker Remote nodes.
 302          */
 303         } else if ((child != NULL) && data->untrusted) {
 304             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 305                                   CRM_DAEMON_DIR "/pacemaker-execd");
 306             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 307                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 308 #endif
 309         } else {
 310             if (data->container_command) {
 311                 crm_create_nvpair_xml(xml_obj, NULL,
 312                                       "run_cmd", data->container_command);
 313             }
 314 
 315             /* TODO: Allow users to specify their own?
 316              *
 317              * We don't know what's in the container, so we just want
 318              * to know if it is alive
 319              */
 320             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 321         }
 322 
 323 
 324         xml_obj = create_xml_node(xml_container, "operations");
 325         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 326 
 327         // TODO: Other ops? Timeouts and intervals from underlying resource?
 328         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 329             return FALSE;
 330         }
 331         parent->children = g_list_append(parent->children, replica->container);
 332         return TRUE;
 333 }
 334 
 335 static bool
 336 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 337                        pe__bundle_replica_t *replica,
 338                        pe_working_set_t *data_set)
 339 {
 340         int offset = 0, max = 4096;
 341         char *buffer = calloc(1, max+1);
 342 
 343         int doffset = 0, dmax = 1024;
 344         char *dbuffer = calloc(1, dmax+1);
 345 
 346         char *id = NULL;
 347         xmlNode *xml_container = NULL;
 348         xmlNode *xml_obj = NULL;
 349 
 350         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
 351         crm_xml_sanitize_id(id);
 352         xml_container = create_resource(id, "heartbeat",
 353                                         PE__CONTAINER_AGENT_PODMAN_S);
 354         free(id);
 355 
 356         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 357         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 358                        data->prefix, replica->offset);
 359 
 360         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 361         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 362         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 363         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 364 
 365         // FIXME: (bandini 2018-08) podman has no restart policies
 366         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 367 
 368         /* Set a container hostname only if we have an IP to map it to.
 369          * The user can set -h or --uts=host themselves if they want a nicer
 370          * name for logs, but this makes applications happy who need their
 371          * hostname to match the IP they bind to.
 372          */
 373         if (data->ip_range_start != NULL) {
 374             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 375                                data->prefix, replica->offset);
 376         }
 377 
 378         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 379 
 380         if (data->container_network) {
 381 #if 0
 382             // podman has no support for --link-local-ip
 383             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 384                                replica->ipaddr);
 385 #endif
 386             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 387                                data->container_network);
 388         }
 389 
 390         if(data->control_port) {
 391             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 392         } else {
 393             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 394         }
 395 
 396         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 397             pe__bundle_mount_t *mount = pIter->data;
 398 
 399             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 400                 char *source = crm_strdup_printf(
 401                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 402 
 403                 if(doffset > 0) {
 404                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 405                 }
 406                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 407                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 408                 free(source);
 409 
 410             } else {
 411                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 412             }
 413             if(mount->options) {
 414                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 415             }
 416         }
 417 
 418         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 419             pe__bundle_port_t *port = pIter->data;
 420 
 421             if (replica->ipaddr) {
 422                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 423                                    replica->ipaddr, port->source,
 424                                    port->target);
 425             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 426                 // No need to do port mapping if net=host
 427                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 428             }
 429         }
 430 
 431         if (data->launcher_options) {
 432             offset += snprintf(buffer+offset, max-offset, " %s",
 433                                data->launcher_options);
 434         }
 435 
 436         if (data->container_host_options) {
 437             offset += snprintf(buffer + offset, max - offset, " %s",
 438                                data->container_host_options);
 439         }
 440 
 441         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 442         free(buffer);
 443 
 444         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 445         free(dbuffer);
 446 
 447         if (replica->child) {
 448             if (data->container_command) {
 449                 crm_create_nvpair_xml(xml_obj, NULL,
 450                                       "run_cmd", data->container_command);
 451             } else {
 452                 crm_create_nvpair_xml(xml_obj, NULL,
 453                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 454             }
 455 
 456             /* TODO: Allow users to specify their own?
 457              *
 458              * We just want to know if the container is alive, we'll
 459              * monitor the child independently
 460              */
 461             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 462 #if 0
 463         /* @TODO Consider supporting the use case where we can start and stop
 464          * resources, but not proxy local commands (such as setting node
 465          * attributes), by running the local executor in stand-alone mode.
 466          * However, this would probably be better done via ACLs as with other
 467          * Pacemaker Remote nodes.
 468          */
 469         } else if ((child != NULL) && data->untrusted) {
 470             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 471                                   CRM_DAEMON_DIR "/pacemaker-execd");
 472             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 473                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 474 #endif
 475         } else {
 476             if (data->container_command) {
 477                 crm_create_nvpair_xml(xml_obj, NULL,
 478                                       "run_cmd", data->container_command);
 479             }
 480 
 481             /* TODO: Allow users to specify their own?
 482              *
 483              * We don't know what's in the container, so we just want
 484              * to know if it is alive
 485              */
 486             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 487         }
 488 
 489 
 490         xml_obj = create_xml_node(xml_container, "operations");
 491         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 492 
 493         // TODO: Other ops? Timeouts and intervals from underlying resource?
 494         if (!common_unpack(xml_container, &replica->container, parent,
 495                            data_set)) {
 496             return FALSE;
 497         }
 498         parent->children = g_list_append(parent->children, replica->container);
 499         return TRUE;
 500 }
 501 
 502 static bool
 503 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 504                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 505 {
 506         int offset = 0, max = 4096;
 507         char *buffer = calloc(1, max+1);
 508 
 509         int doffset = 0, dmax = 1024;
 510         char *dbuffer = calloc(1, dmax+1);
 511 
 512         char *id = NULL;
 513         xmlNode *xml_container = NULL;
 514         xmlNode *xml_obj = NULL;
 515 
 516         int volid = 0;
 517 
 518         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
 519         crm_xml_sanitize_id(id);
 520         xml_container = create_resource(id, "heartbeat",
 521                                         PE__CONTAINER_AGENT_RKT_S);
 522         free(id);
 523 
 524         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 525         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 526                        data->prefix, replica->offset);
 527 
 528         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 529         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
 530         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
 531         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 532 
 533         /* Set a container hostname only if we have an IP to map it to.
 534          * The user can set -h or --uts=host themselves if they want a nicer
 535          * name for logs, but this makes applications happy who need their
 536          * hostname to match the IP they bind to.
 537          */
 538         if (data->ip_range_start != NULL) {
 539             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
 540                                data->prefix, replica->offset);
 541         }
 542 
 543         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 544 
 545         if (data->container_network) {
 546 #if 0
 547             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 548                                replica->ipaddr);
 549 #endif
 550             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 551                                data->container_network);
 552         }
 553 
 554         if(data->control_port) {
 555             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
 556         } else {
 557             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 558         }
 559 
 560         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 561             pe__bundle_mount_t *mount = pIter->data;
 562 
 563             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 564                 char *source = crm_strdup_printf(
 565                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 566 
 567                 if(doffset > 0) {
 568                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 569                 }
 570                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 571                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
 572                 if(mount->options) {
 573                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 574                 }
 575                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 576                 free(source);
 577 
 578             } else {
 579                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
 580                 if(mount->options) {
 581                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 582                 }
 583                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 584             }
 585             volid++;
 586         }
 587 
 588         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 589             pe__bundle_port_t *port = pIter->data;
 590 
 591             if (replica->ipaddr) {
 592                 offset += snprintf(buffer+offset, max-offset,
 593                                    " --port=%s:%s:%s", port->target,
 594                                    replica->ipaddr, port->source);
 595             } else {
 596                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
 597             }
 598         }
 599 
 600         if (data->launcher_options) {
 601             offset += snprintf(buffer+offset, max-offset, " %s",
 602                                data->launcher_options);
 603         }
 604 
 605         if (data->container_host_options) {
 606             offset += snprintf(buffer + offset, max - offset, " %s",
 607                                data->container_host_options);
 608         }
 609 
 610         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 611         free(buffer);
 612 
 613         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 614         free(dbuffer);
 615 
 616         if (replica->child) {
 617             if (data->container_command) {
 618                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 619                                       data->container_command);
 620             } else {
 621                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 622                                       SBIN_DIR "/pacemaker-remoted");
 623             }
 624 
 625             /* TODO: Allow users to specify their own?
 626              *
 627              * We just want to know if the container is alive, we'll
 628              * monitor the child independently
 629              */
 630             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 631 #if 0
 632         /* @TODO Consider supporting the use case where we can start and stop
 633          * resources, but not proxy local commands (such as setting node
 634          * attributes), by running the local executor in stand-alone mode.
 635          * However, this would probably be better done via ACLs as with other
 636          * Pacemaker Remote nodes.
 637          */
 638         } else if ((child != NULL) && data->untrusted) {
 639             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 640                                   CRM_DAEMON_DIR "/pacemaker-execd");
 641             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 642                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 643 #endif
 644         } else {
 645             if (data->container_command) {
 646                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 647                                       data->container_command);
 648             }
 649 
 650             /* TODO: Allow users to specify their own?
 651              *
 652              * We don't know what's in the container, so we just want
 653              * to know if it is alive
 654              */
 655             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 656         }
 657 
 658 
 659         xml_obj = create_xml_node(xml_container, "operations");
 660         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 661 
 662         // TODO: Other ops? Timeouts and intervals from underlying resource?
 663 
 664         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 665             return FALSE;
 666         }
 667         parent->children = g_list_append(parent->children, replica->container);
 668         return TRUE;
 669 }
 670 
 671 /*!
 672  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 673  *
 674  * \param[in,out] rsc    Resource to modify
 675  * \param[in]     uname  Name of node to ban
 676  */
 677 static void
 678 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 679 {
 680     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 681 
 682     if (match) {
 683         ((pe_node_t *) match)->weight = -INFINITY;
 684         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 685     }
 686     if (rsc->children) {
 687         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 688     }
 689 }
 690 
 691 static bool
 692 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 693                        pe__bundle_replica_t *replica,
 694                        pe_working_set_t *data_set)
 695 {
 696     if (replica->child && valid_network(data)) {
 697         GHashTableIter gIter;
 698         pe_node_t *node = NULL;
 699         xmlNode *xml_remote = NULL;
 700         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 701         char *port_s = NULL;
 702         const char *uname = NULL;
 703         const char *connect_name = NULL;
 704 
 705         if (pe_find_resource(data_set->resources, id) != NULL) {
 706             free(id);
 707             // The biggest hammer we have
 708             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 709                                    replica->child->id, replica->offset);
 710             //@TODO return false instead of asserting?
 711             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
 712         }
 713 
 714         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 715          * connection does not have its own IP is a magic string that we use to
 716          * support nested remotes (i.e. a bundle running on a remote node).
 717          */
 718         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 719 
 720         if (data->control_port == NULL) {
 721             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 722         }
 723 
 724         /* This sets replica->container as replica->remote's container, which is
 725          * similar to what happens with guest nodes. This is how the scheduler
 726          * knows that the bundle node is fenced by recovering the container, and
 727          * that remote should be ordered relative to the container.
 728          */
 729         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 730                                           NULL, NULL, NULL,
 731                                           connect_name, (data->control_port?
 732                                           data->control_port : port_s));
 733         free(port_s);
 734 
 735         /* Abandon our created ID, and pull the copy from the XML, because we
 736          * need something that will get freed during data set cleanup to use as
 737          * the node ID and uname.
 738          */
 739         free(id);
 740         id = NULL;
 741         uname = ID(xml_remote);
 742 
 743         /* Ensure a node has been created for the guest (it may have already
 744          * been, if it has a permanent node attribute), and ensure its weight is
 745          * -INFINITY so no other resources can run on it.
 746          */
 747         node = pe_find_node(data_set->nodes, uname);
 748         if (node == NULL) {
 749             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 750                                   data_set);
 751         } else {
 752             node->weight = -INFINITY;
 753         }
 754         node->rsc_discover_mode = pe_discover_never;
 755 
 756         /* unpack_remote_nodes() ensures that each remote node and guest node
 757          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 758          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 759          * what nodes will be needed, so we do it just above.
 760          *
 761          * Worse, that means that the node may have been utilized while
 762          * unpacking other resources, without our weight correction. The most
 763          * likely place for this to happen is when common_unpack() calls
 764          * resource_location() to set a default score in symmetric clusters.
 765          * This adds a node *copy* to each resource's allowed nodes, and these
 766          * copies will have the wrong weight.
 767          *
 768          * As a hacky workaround, fix those copies here.
 769          *
 770          * @TODO Possible alternative: ensure bundles are unpacked before other
 771          * resources, so the weight is correct before any copies are made.
 772          */
 773         g_list_foreach(data_set->resources, (GFunc) disallow_node, (gpointer) uname);
 774 
 775         replica->node = pe__copy_node(node);
 776         replica->node->weight = 500;
 777         replica->node->rsc_discover_mode = pe_discover_exclusive;
 778 
 779         /* Ensure the node shows up as allowed and with the correct discovery set */
 780         if (replica->child->allowed_nodes != NULL) {
 781             g_hash_table_destroy(replica->child->allowed_nodes);
 782         }
 783         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 784         g_hash_table_insert(replica->child->allowed_nodes,
 785                             (gpointer) replica->node->details->id,
 786                             pe__copy_node(replica->node));
 787 
 788         {
 789             pe_node_t *copy = pe__copy_node(replica->node);
 790             copy->weight = -INFINITY;
 791             g_hash_table_insert(replica->child->parent->allowed_nodes,
 792                                 (gpointer) replica->node->details->id, copy);
 793         }
 794         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
 795             return FALSE;
 796         }
 797 
 798         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 799         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 800             if (pe__is_guest_or_remote_node(node)) {
 801                 /* Remote resources can only run on 'normal' cluster node */
 802                 node->weight = -INFINITY;
 803             }
 804         }
 805 
 806         replica->node->details->remote_rsc = replica->remote;
 807 
 808         // Ensure pe__is_guest_node() functions correctly immediately
 809         replica->remote->container = replica->container;
 810 
 811         /* A bundle's #kind is closer to "container" (guest node) than the
 812          * "remote" set by pe_create_node().
 813          */
 814         g_hash_table_insert(replica->node->details->attrs,
 815                             strdup(CRM_ATTR_KIND), strdup("container"));
 816 
 817         /* One effect of this is that setup_container() will add
 818          * replica->remote to replica->container's fillers, which will make
 819          * pe__resource_contains_guest_node() true for replica->container.
 820          *
 821          * replica->child does NOT get added to replica->container's fillers.
 822          * The only noticeable effect if it did would be for its fail count to
 823          * be taken into account when checking replica->container's migration
 824          * threshold.
 825          */
 826         parent->children = g_list_append(parent->children, replica->remote);
 827     }
 828     return TRUE;
 829 }
 830 
 831 static bool
 832 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 833                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 834 {
 835 
 836     switch (data->agent_type) {
 837         case PE__CONTAINER_AGENT_DOCKER:
 838             if (!create_docker_resource(parent, data, replica, data_set)) {
 839                 return FALSE;
 840             }
 841             break;
 842 
 843         case PE__CONTAINER_AGENT_PODMAN:
 844             if (!create_podman_resource(parent, data, replica, data_set)) {
 845                 return FALSE;
 846             }
 847             break;
 848 
 849         case PE__CONTAINER_AGENT_RKT:
 850             if (!create_rkt_resource(parent, data, replica, data_set)) {
 851                 return FALSE;
 852             }
 853             break;
 854         default: // PE__CONTAINER_AGENT_UNKNOWN
 855             return FALSE;
 856     }
 857 
 858     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
 859         return FALSE;
 860     }
 861     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
 862         return FALSE;
 863     }
 864     if (replica->child && replica->ipaddr) {
 865         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 866     }
 867 
 868     if (replica->remote) {
 869         /*
 870          * Allow the remote connection resource to be allocated to a
 871          * different node than the one on which the container is active.
 872          *
 873          * This makes it possible to have Pacemaker Remote nodes running
 874          * containers with pacemaker-remoted inside in order to start
 875          * services inside those containers.
 876          */
 877         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 878     }
 879 
 880     return TRUE;
 881 }
 882 
 883 static void
 884 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 885           const char *target, const char *options, uint32_t flags)
 886 {
 887     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 888 
 889     mount->source = strdup(source);
 890     mount->target = strdup(target);
 891     if (options) {
 892         mount->options = strdup(options);
 893     }
 894     mount->flags = flags;
 895     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 896 }
 897 
 898 static void
 899 mount_free(pe__bundle_mount_t *mount)
     /*  */
 900 {
 901     free(mount->source);
 902     free(mount->target);
 903     free(mount->options);
 904     free(mount);
 905 }
 906 
 907 static void
 908 port_free(pe__bundle_port_t *port)
     /*  */
 909 {
 910     free(port->source);
 911     free(port->target);
 912     free(port);
 913 }
 914 
 915 static pe__bundle_replica_t *
 916 replica_for_remote(pe_resource_t *remote)
     /*  */
 917 {
 918     pe_resource_t *top = remote;
 919     pe__bundle_variant_data_t *bundle_data = NULL;
 920 
 921     if (top == NULL) {
 922         return NULL;
 923     }
 924 
 925     while (top->parent != NULL) {
 926         top = top->parent;
 927     }
 928 
 929     get_bundle_variant_data(bundle_data, top);
 930     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 931          gIter = gIter->next) {
 932         pe__bundle_replica_t *replica = gIter->data;
 933 
 934         if (replica->remote == remote) {
 935             return replica;
 936         }
 937     }
 938     CRM_LOG_ASSERT(FALSE);
 939     return NULL;
 940 }
 941 
 942 bool
 943 pe__bundle_needs_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 944 {
 945     const char *value;
 946     GHashTable *params = NULL;
 947 
 948     if (rsc == NULL) {
 949         return false;
 950     }
 951 
 952     // Use NULL node since pcmk__bundle_expand() uses that to set value
 953     params = pe_rsc_params(rsc, NULL, data_set);
 954     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 955 
 956     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 957            && xml_contains_remote_node(rsc->xml);
 958 }
 959 
 960 const char *
 961 pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
     /*  */
 962                            xmlNode *xml, const char *field)
 963 {
 964     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 965 
 966     pe_node_t *node = NULL;
 967     pe__bundle_replica_t *replica = NULL;
 968 
 969     if (!pe__bundle_needs_remote_name(rsc, data_set)) {
 970         return NULL;
 971     }
 972 
 973     replica = replica_for_remote(rsc);
 974     if (replica == NULL) {
 975         return NULL;
 976     }
 977 
 978     node = replica->container->allocated_to;
 979     if (node == NULL) {
 980         /* If it won't be running anywhere after the
 981          * transition, go with where it's running now.
 982          */
 983         node = pe__current_node(replica->container);
 984     }
 985 
 986     if(node == NULL) {
 987         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 988         return NULL;
 989     }
 990 
 991     crm_trace("Setting address for bundle connection %s to bundle host %s",
 992               rsc->id, node->details->uname);
 993     if(xml != NULL && field != NULL) {
 994         crm_xml_add(xml, field, node->details->uname);
 995     }
 996 
 997     return node->details->uname;
 998 }
 999 
1000 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
1001         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
1002                                    "Bundle mount", ID(mount_xml), flags,    \
1003                                    (flags_to_set), #flags_to_set);          \
1004     } while (0)
1005 
1006 gboolean
1007 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
1008 {
1009     const char *value = NULL;
1010     xmlNode *xml_obj = NULL;
1011     xmlNode *xml_resource = NULL;
1012     pe__bundle_variant_data_t *bundle_data = NULL;
1013     bool need_log_mount = TRUE;
1014 
1015     CRM_ASSERT(rsc != NULL);
1016     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
1017 
1018     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
1019     rsc->variant_opaque = bundle_data;
1020     bundle_data->prefix = strdup(rsc->id);
1021 
1022     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1023     if (xml_obj != NULL) {
1024         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1025     } else {
1026         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1027         if (xml_obj != NULL) {
1028             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1029         } else {
1030             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1031             if (xml_obj != NULL) {
1032                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1033             } else {
1034                 return FALSE;
1035             }
1036         }
1037     }
1038 
1039     // Use 0 for default, minimum, and invalid promoted-max
1040     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
1041     if (value == NULL) {
1042         // @COMPAT deprecated since 2.0.0
1043         value = crm_element_value(xml_obj, "masters");
1044     }
1045     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1046 
1047     // Default replicas to promoted-max if it was specified and 1 otherwise
1048     value = crm_element_value(xml_obj, "replicas");
1049     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1050         bundle_data->nreplicas = bundle_data->promoted_max;
1051     } else {
1052         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1053     }
1054 
1055     /*
1056      * Communication between containers on the same host via the
1057      * floating IPs only works if the container is started with:
1058      *   --userland-proxy=false --ip-masq=false
1059      */
1060     value = crm_element_value(xml_obj, "replicas-per-host");
1061     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1062     if (bundle_data->nreplicas_per_host == 1) {
1063         pe__clear_resource_flags(rsc, pe_rsc_unique);
1064     }
1065 
1066     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1067     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1068     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1069     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1070 
1071     xml_obj = first_named_child(rsc->xml, "network");
1072     if(xml_obj) {
1073 
1074         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1075         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1076         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1077         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1078         value = crm_element_value(xml_obj, "add-host");
1079         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1080             bundle_data->add_host = TRUE;
1081         }
1082 
1083         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1084              xml_child = pcmk__xe_next(xml_child)) {
1085 
1086             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1087             port->source = crm_element_value_copy(xml_child, "port");
1088 
1089             if(port->source == NULL) {
1090                 port->source = crm_element_value_copy(xml_child, "range");
1091             } else {
1092                 port->target = crm_element_value_copy(xml_child, "internal-port");
1093             }
1094 
1095             if(port->source != NULL && strlen(port->source) > 0) {
1096                 if(port->target == NULL) {
1097                     port->target = strdup(port->source);
1098                 }
1099                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1100 
1101             } else {
1102                 pe_err("Invalid port directive %s", ID(xml_child));
1103                 port_free(port);
1104             }
1105         }
1106     }
1107 
1108     xml_obj = first_named_child(rsc->xml, "storage");
1109     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1110          xml_child = pcmk__xe_next(xml_child)) {
1111 
1112         const char *source = crm_element_value(xml_child, "source-dir");
1113         const char *target = crm_element_value(xml_child, "target-dir");
1114         const char *options = crm_element_value(xml_child, "options");
1115         int flags = pe__bundle_mount_none;
1116 
1117         if (source == NULL) {
1118             source = crm_element_value(xml_child, "source-dir-root");
1119             pe__set_bundle_mount_flags(xml_child, flags,
1120                                        pe__bundle_mount_subdir);
1121         }
1122 
1123         if (source && target) {
1124             mount_add(bundle_data, source, target, options, flags);
1125             if (strcmp(target, "/var/log") == 0) {
1126                 need_log_mount = FALSE;
1127             }
1128         } else {
1129             pe_err("Invalid mount directive %s", ID(xml_child));
1130         }
1131     }
1132 
1133     xml_obj = first_named_child(rsc->xml, "primitive");
1134     if (xml_obj && valid_network(bundle_data)) {
1135         char *value = NULL;
1136         xmlNode *xml_set = NULL;
1137 
1138         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1139 
1140         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1141          * part of the resource name, so that bundles don't restart in a rolling
1142          * upgrade. (It also avoids needing to change regression tests.)
1143          */
1144         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1145                       (bundle_data->promoted_max? "master"
1146                       : (const char *)xml_resource->name));
1147 
1148         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1149         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1150 
1151         crm_create_nvpair_xml(xml_set, NULL,
1152                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1153 
1154         value = pcmk__itoa(bundle_data->nreplicas);
1155         crm_create_nvpair_xml(xml_set, NULL,
1156                               XML_RSC_ATTR_INCARNATION_MAX, value);
1157         free(value);
1158 
1159         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1160         crm_create_nvpair_xml(xml_set, NULL,
1161                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
1162         free(value);
1163 
1164         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1165                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1166 
1167         if (bundle_data->promoted_max) {
1168             crm_create_nvpair_xml(xml_set, NULL,
1169                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1170 
1171             value = pcmk__itoa(bundle_data->promoted_max);
1172             crm_create_nvpair_xml(xml_set, NULL,
1173                                   XML_RSC_ATTR_PROMOTED_MAX, value);
1174             free(value);
1175         }
1176 
1177         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1178         add_node_copy(xml_resource, xml_obj);
1179 
1180     } else if(xml_obj) {
1181         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1182                rsc->id, ID(xml_obj));
1183         return FALSE;
1184     }
1185 
1186     if(xml_resource) {
1187         int lpc = 0;
1188         GList *childIter = NULL;
1189         pe_resource_t *new_rsc = NULL;
1190         pe__bundle_port_t *port = NULL;
1191 
1192         int offset = 0, max = 1024;
1193         char *buffer = NULL;
1194 
1195         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
1196             pe_err("Failed unpacking resource %s", ID(rsc->xml));
1197             if (new_rsc != NULL && new_rsc->fns != NULL) {
1198                 new_rsc->fns->free(new_rsc);
1199             }
1200             return FALSE;
1201         }
1202 
1203         bundle_data->child = new_rsc;
1204 
1205         /* Currently, we always map the default authentication key location
1206          * into the same location inside the container.
1207          *
1208          * Ideally, we would respect the host's PCMK_authkey_location, but:
1209          * - it may be different on different nodes;
1210          * - the actual connection will do extra checking to make sure the key
1211          *   file exists and is readable, that we can't do here on the DC
1212          * - tools such as crm_resource and crm_simulate may not have the same
1213          *   environment variables as the cluster, causing operation digests to
1214          *   differ
1215          *
1216          * Always using the default location inside the container is fine,
1217          * because we control the pacemaker_remote environment, and it avoids
1218          * having to pass another environment variable to the container.
1219          *
1220          * @TODO A better solution may be to have only pacemaker_remote use the
1221          * environment variable, and have the cluster nodes use a new
1222          * cluster option for key location. This would introduce the limitation
1223          * of the location being the same on all cluster nodes, but that's
1224          * reasonable.
1225          */
1226         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1227                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1228 
1229         if (need_log_mount) {
1230             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1231                       pe__bundle_mount_subdir);
1232         }
1233 
1234         port = calloc(1, sizeof(pe__bundle_port_t));
1235         if(bundle_data->control_port) {
1236             port->source = strdup(bundle_data->control_port);
1237         } else {
1238             /* If we wanted to respect PCMK_remote_port, we could use
1239              * crm_default_remote_port() here and elsewhere in this file instead
1240              * of DEFAULT_REMOTE_PORT.
1241              *
1242              * However, it gains nothing, since we control both the container
1243              * environment and the connection resource parameters, and the user
1244              * can use a different port if desired by setting control-port.
1245              */
1246             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1247         }
1248         port->target = strdup(port->source);
1249         bundle_data->ports = g_list_append(bundle_data->ports, port);
1250 
1251         buffer = calloc(1, max+1);
1252         for (childIter = bundle_data->child->children; childIter != NULL;
1253              childIter = childIter->next) {
1254 
1255             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1256 
1257             replica->child = childIter->data;
1258             replica->child->exclusive_discover = TRUE;
1259             replica->offset = lpc++;
1260 
1261             // Ensure the child's notify gets set based on the underlying primitive's value
1262             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1263                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1264             }
1265 
1266             offset += allocate_ip(bundle_data, replica, buffer+offset,
1267                                   max-offset);
1268             bundle_data->replicas = g_list_append(bundle_data->replicas,
1269                                                   replica);
1270             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1271                                                                 XML_RSC_ATTR_TARGET);
1272         }
1273         bundle_data->container_host_options = buffer;
1274         if (bundle_data->attribute_target) {
1275             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1276                                  strdup(bundle_data->attribute_target));
1277             g_hash_table_replace(bundle_data->child->meta,
1278                                  strdup(XML_RSC_ATTR_TARGET),
1279                                  strdup(bundle_data->attribute_target));
1280         }
1281 
1282     } else {
1283         // Just a naked container, no pacemaker-remote
1284         int offset = 0, max = 1024;
1285         char *buffer = calloc(1, max+1);
1286 
1287         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1288             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1289 
1290             replica->offset = lpc;
1291             offset += allocate_ip(bundle_data, replica, buffer+offset,
1292                                   max-offset);
1293             bundle_data->replicas = g_list_append(bundle_data->replicas,
1294                                                   replica);
1295         }
1296         bundle_data->container_host_options = buffer;
1297     }
1298 
1299     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1300          gIter = gIter->next) {
1301         pe__bundle_replica_t *replica = gIter->data;
1302 
1303         if (!create_container(rsc, bundle_data, replica, data_set)) {
1304             pe_err("Failed unpacking resource %s", rsc->id);
1305             rsc->fns->free(rsc);
1306             return FALSE;
1307         }
1308     }
1309 
1310     if (bundle_data->child) {
1311         rsc->children = g_list_append(rsc->children, bundle_data->child);
1312     }
1313     return TRUE;
1314 }
1315 
1316 static int
1317 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1318 {
1319     if (rsc) {
1320         gboolean child_active = rsc->fns->active(rsc, all);
1321 
1322         if (child_active && !all) {
1323             return TRUE;
1324         } else if (!child_active && all) {
1325             return FALSE;
1326         }
1327     }
1328     return -1;
1329 }
1330 
1331 gboolean
1332 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1333 {
1334     pe__bundle_variant_data_t *bundle_data = NULL;
1335     GList *iter = NULL;
1336 
1337     get_bundle_variant_data(bundle_data, rsc);
1338     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1339         pe__bundle_replica_t *replica = iter->data;
1340         int rsc_active;
1341 
1342         rsc_active = replica_resource_active(replica->ip, all);
1343         if (rsc_active >= 0) {
1344             return (gboolean) rsc_active;
1345         }
1346 
1347         rsc_active = replica_resource_active(replica->child, all);
1348         if (rsc_active >= 0) {
1349             return (gboolean) rsc_active;
1350         }
1351 
1352         rsc_active = replica_resource_active(replica->container, all);
1353         if (rsc_active >= 0) {
1354             return (gboolean) rsc_active;
1355         }
1356 
1357         rsc_active = replica_resource_active(replica->remote, all);
1358         if (rsc_active >= 0) {
1359             return (gboolean) rsc_active;
1360         }
1361     }
1362 
1363     /* If "all" is TRUE, we've already checked that no resources were inactive,
1364      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1365      * so return FALSE.
1366      */
1367     return all;
1368 }
1369 
1370 /*!
1371  * \internal
1372  * \brief Find the bundle replica corresponding to a given node
1373  *
1374  * \param[in] bundle  Top-level bundle resource
1375  * \param[in] node    Node to search for
1376  *
1377  * \return Bundle replica if found, NULL otherwise
1378  */
1379 pe_resource_t *
1380 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1381 {
1382     pe__bundle_variant_data_t *bundle_data = NULL;
1383     CRM_ASSERT(bundle && node);
1384 
1385     get_bundle_variant_data(bundle_data, bundle);
1386     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1387          gIter = gIter->next) {
1388         pe__bundle_replica_t *replica = gIter->data;
1389 
1390         CRM_ASSERT(replica && replica->node);
1391         if (replica->node->details == node->details) {
1392             return replica->child;
1393         }
1394     }
1395     return NULL;
1396 }
1397 
1398 static void
1399 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1400                   void *print_data)
1401 {
1402     if (rsc != NULL) {
1403         if (options & pe_print_html) {
1404             status_print("<li>");
1405         }
1406         rsc->fns->print(rsc, pre_text, options, print_data);
1407         if (options & pe_print_html) {
1408             status_print("</li>\n");
1409         }
1410     }
1411 }
1412 
1413 static const char*
1414 container_agent_str(enum pe__container_agent t)
     /*  */
1415 {
1416     switch (t) {
1417         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
1418         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
1419         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
1420         default: // PE__CONTAINER_AGENT_UNKNOWN
1421             break;
1422     }
1423     return PE__CONTAINER_AGENT_UNKNOWN_S;
1424 }
1425 
1426 static void
1427 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1428                  void *print_data)
1429 {
1430     pe__bundle_variant_data_t *bundle_data = NULL;
1431     char *child_text = NULL;
1432     CRM_CHECK(rsc != NULL, return);
1433 
1434     if (pre_text == NULL) {
1435         pre_text = "";
1436     }
1437     child_text = crm_strdup_printf("%s        ", pre_text);
1438 
1439     get_bundle_variant_data(bundle_data, rsc);
1440 
1441     status_print("%s<bundle ", pre_text);
1442     status_print("id=\"%s\" ", rsc->id);
1443     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1444     status_print("image=\"%s\" ", bundle_data->image);
1445     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1446     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1447     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1448     status_print(">\n");
1449 
1450     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1451          gIter = gIter->next) {
1452         pe__bundle_replica_t *replica = gIter->data;
1453 
1454         CRM_ASSERT(replica);
1455         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
1456         print_rsc_in_list(replica->ip, child_text, options, print_data);
1457         print_rsc_in_list(replica->child, child_text, options, print_data);
1458         print_rsc_in_list(replica->container, child_text, options, print_data);
1459         print_rsc_in_list(replica->remote, child_text, options, print_data);
1460         status_print("%s    </replica>\n", pre_text);
1461     }
1462     status_print("%s</bundle>\n", pre_text);
1463     free(child_text);
1464 }
1465 
1466 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1467 int
1468 pe__bundle_xml(pcmk__output_t *out, va_list args)
1469 {
1470     unsigned int show_opts = va_arg(args, unsigned int);
1471     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1472     GList *only_node = va_arg(args, GList *);
1473     GList *only_rsc = va_arg(args, GList *);
1474 
1475     pe__bundle_variant_data_t *bundle_data = NULL;
1476     int rc = pcmk_rc_no_output;
1477     gboolean printed_header = FALSE;
1478     gboolean print_everything = TRUE;
1479 
1480     CRM_ASSERT(rsc != NULL);
1481 
1482     get_bundle_variant_data(bundle_data, rsc);
1483 
1484     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1485         return rc;
1486     }
1487 
1488     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1489 
1490     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1491          gIter = gIter->next) {
1492         pe__bundle_replica_t *replica = gIter->data;
1493         char *id = NULL;
1494         gboolean print_ip, print_child, print_ctnr, print_remote;
1495 
1496         CRM_ASSERT(replica);
1497 
1498         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1499             continue;
1500         }
1501 
1502         print_ip = replica->ip != NULL &&
1503                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1504         print_child = replica->child != NULL &&
1505                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1506         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1507         print_remote = replica->remote != NULL &&
1508                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1509 
1510         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1511             continue;
1512         }
1513 
1514         if (!printed_header) {
1515             printed_header = TRUE;
1516 
1517             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6,
1518                      "id", rsc->id,
1519                      "type", container_agent_str(bundle_data->agent_type),
1520                      "image", bundle_data->image,
1521                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1522                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1523                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed));
1524             CRM_ASSERT(rc == pcmk_rc_ok);
1525         }
1526 
1527         id = pcmk__itoa(replica->offset);
1528         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1529         free(id);
1530         CRM_ASSERT(rc == pcmk_rc_ok);
1531 
1532         if (print_ip) {
1533             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
1534                          replica->ip, only_node, only_rsc);
1535         }
1536 
1537         if (print_child) {
1538             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
1539                          replica->child, only_node, only_rsc);
1540         }
1541 
1542         if (print_ctnr) {
1543             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
1544                          replica->container, only_node, only_rsc);
1545         }
1546 
1547         if (print_remote) {
1548             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
1549                          replica->remote, only_node, only_rsc);
1550         }
1551 
1552         pcmk__output_xml_pop_parent(out); // replica
1553     }
1554 
1555     if (printed_header) {
1556         pcmk__output_xml_pop_parent(out); // bundle
1557     }
1558 
1559     return rc;
1560 }
1561 
1562 static void
1563 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1564                                pe_node_t *node, unsigned int show_opts)
1565 {
1566     pe_resource_t *rsc = replica->child;
1567 
1568     int offset = 0;
1569     char buffer[LINE_MAX];
1570 
1571     if(rsc == NULL) {
1572         rsc = replica->container;
1573     }
1574 
1575     if (replica->remote) {
1576         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1577                            rsc_printable_id(replica->remote));
1578     } else {
1579         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1580                            rsc_printable_id(replica->container));
1581     }
1582     if (replica->ipaddr) {
1583         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1584                            replica->ipaddr);
1585     }
1586 
1587     pe__common_output_html(out, rsc, buffer, node, show_opts);
1588 }
1589 
1590 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1591 int
1592 pe__bundle_html(pcmk__output_t *out, va_list args)
1593 {
1594     unsigned int show_opts = va_arg(args, unsigned int);
1595     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1596     GList *only_node = va_arg(args, GList *);
1597     GList *only_rsc = va_arg(args, GList *);
1598 
1599     pe__bundle_variant_data_t *bundle_data = NULL;
1600     int rc = pcmk_rc_no_output;
1601     gboolean print_everything = TRUE;
1602 
1603     CRM_ASSERT(rsc != NULL);
1604 
1605     get_bundle_variant_data(bundle_data, rsc);
1606 
1607     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1608         return rc;
1609     }
1610 
1611     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1612 
1613     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1614          gIter = gIter->next) {
1615         pe__bundle_replica_t *replica = gIter->data;
1616         gboolean print_ip, print_child, print_ctnr, print_remote;
1617 
1618         CRM_ASSERT(replica);
1619 
1620         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1621             continue;
1622         }
1623 
1624         print_ip = replica->ip != NULL &&
1625                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1626         print_child = replica->child != NULL &&
1627                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1628         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1629         print_remote = replica->remote != NULL &&
1630                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1631 
1632         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1633             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1634             /* The text output messages used below require pe_print_implicit to
1635              * be set to do anything.
1636              */
1637             unsigned int new_show_opts = show_opts | pcmk_show_implicit_rscs;
1638 
1639             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1640                                      (bundle_data->nreplicas > 1)? " set" : "",
1641                                      rsc->id, bundle_data->image,
1642                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1643                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1644 
1645             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1646                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1647             }
1648 
1649             if (print_ip) {
1650                 out->message(out, crm_map_element_name(replica->ip->xml),
1651                              new_show_opts, replica->ip, only_node, only_rsc);
1652             }
1653 
1654             if (print_child) {
1655                 out->message(out, crm_map_element_name(replica->child->xml),
1656                              new_show_opts, replica->child, only_node, only_rsc);
1657             }
1658 
1659             if (print_ctnr) {
1660                 out->message(out, crm_map_element_name(replica->container->xml),
1661                              new_show_opts, replica->container, only_node, only_rsc);
1662             }
1663 
1664             if (print_remote) {
1665                 out->message(out, crm_map_element_name(replica->remote->xml),
1666                              new_show_opts, replica->remote, only_node, only_rsc);
1667             }
1668 
1669             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1670                 out->end_list(out);
1671             }
1672         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1673             continue;
1674         } else {
1675             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1676                                      (bundle_data->nreplicas > 1)? " set" : "",
1677                                      rsc->id, bundle_data->image,
1678                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1679                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1680 
1681             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1682                                            show_opts);
1683         }
1684     }
1685 
1686     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1687     return rc;
1688 }
1689 
1690 static void
1691 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1692                                pe_node_t *node, unsigned int show_opts)
1693 {
1694     pe_resource_t *rsc = replica->child;
1695 
1696     int offset = 0;
1697     char buffer[LINE_MAX];
1698 
1699     if(rsc == NULL) {
1700         rsc = replica->container;
1701     }
1702 
1703     if (replica->remote) {
1704         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1705                            rsc_printable_id(replica->remote));
1706     } else {
1707         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1708                            rsc_printable_id(replica->container));
1709     }
1710     if (replica->ipaddr) {
1711         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1712                            replica->ipaddr);
1713     }
1714 
1715     pe__common_output_text(out, rsc, buffer, node, show_opts);
1716 }
1717 
1718 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1719 int
1720 pe__bundle_text(pcmk__output_t *out, va_list args)
1721 {
1722     unsigned int show_opts = va_arg(args, unsigned int);
1723     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1724     GList *only_node = va_arg(args, GList *);
1725     GList *only_rsc = va_arg(args, GList *);
1726 
1727     pe__bundle_variant_data_t *bundle_data = NULL;
1728     int rc = pcmk_rc_no_output;
1729     gboolean print_everything = TRUE;
1730 
1731     get_bundle_variant_data(bundle_data, rsc);
1732 
1733     CRM_ASSERT(rsc != NULL);
1734 
1735     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1736         return rc;
1737     }
1738 
1739     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1740 
1741     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1742          gIter = gIter->next) {
1743         pe__bundle_replica_t *replica = gIter->data;
1744         gboolean print_ip, print_child, print_ctnr, print_remote;
1745 
1746         CRM_ASSERT(replica);
1747 
1748         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1749             continue;
1750         }
1751 
1752         print_ip = replica->ip != NULL &&
1753                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1754         print_child = replica->child != NULL &&
1755                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1756         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1757         print_remote = replica->remote != NULL &&
1758                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1759 
1760         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1761             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1762             /* The text output messages used below require pe_print_implicit to
1763              * be set to do anything.
1764              */
1765             unsigned int new_show_opts = show_opts | pcmk_show_implicit_rscs;
1766 
1767             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1768                                      (bundle_data->nreplicas > 1)? " set" : "",
1769                                      rsc->id, bundle_data->image,
1770                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1771                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1772 
1773             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1774                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1775             }
1776 
1777             out->begin_list(out, NULL, NULL, NULL);
1778 
1779             if (print_ip) {
1780                 out->message(out, crm_map_element_name(replica->ip->xml),
1781                              new_show_opts, replica->ip, only_node, only_rsc);
1782             }
1783 
1784             if (print_child) {
1785                 out->message(out, crm_map_element_name(replica->child->xml),
1786                              new_show_opts, replica->child, only_node, only_rsc);
1787             }
1788 
1789             if (print_ctnr) {
1790                 out->message(out, crm_map_element_name(replica->container->xml),
1791                              new_show_opts, replica->container, only_node, only_rsc);
1792             }
1793 
1794             if (print_remote) {
1795                 out->message(out, crm_map_element_name(replica->remote->xml),
1796                              new_show_opts, replica->remote, only_node, only_rsc);
1797             }
1798 
1799             out->end_list(out);
1800         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1801             continue;
1802         } else {
1803             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1804                                      (bundle_data->nreplicas > 1)? " set" : "",
1805                                      rsc->id, bundle_data->image,
1806                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1807                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1808 
1809             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1810                                            show_opts);
1811         }
1812     }
1813 
1814     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1815     return rc;
1816 }
1817 
1818 static void
1819 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1820                      long options, void *print_data)
1821 {
1822     pe_node_t *node = NULL;
1823     pe_resource_t *rsc = replica->child;
1824 
1825     int offset = 0;
1826     char buffer[LINE_MAX];
1827 
1828     if(rsc == NULL) {
1829         rsc = replica->container;
1830     }
1831 
1832     if (replica->remote) {
1833         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1834                            rsc_printable_id(replica->remote));
1835     } else {
1836         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1837                            rsc_printable_id(replica->container));
1838     }
1839     if (replica->ipaddr) {
1840         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1841                            replica->ipaddr);
1842     }
1843 
1844     node = pe__current_node(replica->container);
1845     common_print(rsc, pre_text, buffer, node, options, print_data);
1846 }
1847 
1848 void
1849 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1850                  void *print_data)
1851 {
1852     pe__bundle_variant_data_t *bundle_data = NULL;
1853     char *child_text = NULL;
1854     CRM_CHECK(rsc != NULL, return);
1855 
1856     if (options & pe_print_xml) {
1857         bundle_print_xml(rsc, pre_text, options, print_data);
1858         return;
1859     }
1860 
1861     get_bundle_variant_data(bundle_data, rsc);
1862 
1863     if (pre_text == NULL) {
1864         pre_text = " ";
1865     }
1866 
1867     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1868                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1869                  rsc->id, bundle_data->image,
1870                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1871                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1872     if (options & pe_print_html) {
1873         status_print("<br />\n<ul>\n");
1874     }
1875 
1876 
1877     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1878          gIter = gIter->next) {
1879         pe__bundle_replica_t *replica = gIter->data;
1880 
1881         CRM_ASSERT(replica);
1882         if (options & pe_print_html) {
1883             status_print("<li>");
1884         }
1885 
1886         if (pcmk_is_set(options, pe_print_implicit)) {
1887             child_text = crm_strdup_printf("     %s", pre_text);
1888             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1889                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1890             }
1891             if (options & pe_print_html) {
1892                 status_print("<br />\n<ul>\n");
1893             }
1894             print_rsc_in_list(replica->ip, child_text, options, print_data);
1895             print_rsc_in_list(replica->container, child_text, options, print_data);
1896             print_rsc_in_list(replica->remote, child_text, options, print_data);
1897             print_rsc_in_list(replica->child, child_text, options, print_data);
1898             if (options & pe_print_html) {
1899                 status_print("</ul>\n");
1900             }
1901         } else {
1902             child_text = crm_strdup_printf("%s  ", pre_text);
1903             print_bundle_replica(replica, child_text, options, print_data);
1904         }
1905         free(child_text);
1906 
1907         if (options & pe_print_html) {
1908             status_print("</li>\n");
1909         }
1910     }
1911     if (options & pe_print_html) {
1912         status_print("</ul>\n");
1913     }
1914 }
1915 
1916 static void
1917 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1918 {
1919     if (replica == NULL) {
1920         return;
1921     }
1922 
1923     if (replica->node) {
1924         free(replica->node);
1925         replica->node = NULL;
1926     }
1927 
1928     if (replica->ip) {
1929         free_xml(replica->ip->xml);
1930         replica->ip->xml = NULL;
1931         replica->ip->fns->free(replica->ip);
1932         replica->ip = NULL;
1933     }
1934     if (replica->container) {
1935         free_xml(replica->container->xml);
1936         replica->container->xml = NULL;
1937         replica->container->fns->free(replica->container);
1938         replica->container = NULL;
1939     }
1940     if (replica->remote) {
1941         free_xml(replica->remote->xml);
1942         replica->remote->xml = NULL;
1943         replica->remote->fns->free(replica->remote);
1944         replica->remote = NULL;
1945     }
1946     free(replica->ipaddr);
1947     free(replica);
1948 }
1949 
1950 void
1951 pe__free_bundle(pe_resource_t *rsc)
     /*  */
1952 {
1953     pe__bundle_variant_data_t *bundle_data = NULL;
1954     CRM_CHECK(rsc != NULL, return);
1955 
1956     get_bundle_variant_data(bundle_data, rsc);
1957     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1958 
1959     free(bundle_data->prefix);
1960     free(bundle_data->image);
1961     free(bundle_data->control_port);
1962     free(bundle_data->host_network);
1963     free(bundle_data->host_netmask);
1964     free(bundle_data->ip_range_start);
1965     free(bundle_data->container_network);
1966     free(bundle_data->launcher_options);
1967     free(bundle_data->container_command);
1968     free(bundle_data->container_host_options);
1969 
1970     g_list_free_full(bundle_data->replicas,
1971                      (GDestroyNotify) free_bundle_replica);
1972     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1973     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1974     g_list_free(rsc->children);
1975 
1976     if(bundle_data->child) {
1977         free_xml(bundle_data->child->xml);
1978         bundle_data->child->xml = NULL;
1979         bundle_data->child->fns->free(bundle_data->child);
1980     }
1981     common_free(rsc);
1982 }
1983 
1984 enum rsc_role_e
1985 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
1986 {
1987     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
1988     return container_role;
1989 }
1990 
1991 /*!
1992  * \brief Get the number of configured replicas in a bundle
1993  *
1994  * \param[in] rsc  Bundle resource
1995  *
1996  * \return Number of configured replicas, or 0 on error
1997  */
1998 int
1999 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
2000 {
2001     if ((rsc == NULL) || (rsc->variant != pe_container)) {
2002         return 0;
2003     } else {
2004         pe__bundle_variant_data_t *bundle_data = NULL;
2005 
2006         get_bundle_variant_data(bundle_data, rsc);
2007         return bundle_data->nreplicas;
2008     }
2009 }
2010 
2011 void
2012 pe__count_bundle(pe_resource_t *rsc)
     /*  */
2013 {
2014     pe__bundle_variant_data_t *bundle_data = NULL;
2015 
2016     get_bundle_variant_data(bundle_data, rsc);
2017     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2018         pe__bundle_replica_t *replica = item->data;
2019 
2020         if (replica->ip) {
2021             replica->ip->fns->count(replica->ip);
2022         }
2023         if (replica->child) {
2024             replica->child->fns->count(replica->child);
2025         }
2026         if (replica->container) {
2027             replica->container->fns->count(replica->container);
2028         }
2029         if (replica->remote) {
2030             replica->remote->fns->count(replica->remote);
2031         }
2032     }
2033 }
2034 
2035 gboolean
2036 pe__bundle_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent)
     /*  */
2037 {
2038     gboolean passes = FALSE;
2039     pe__bundle_variant_data_t *bundle_data = NULL;
2040 
2041     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
2042         passes = TRUE;
2043     } else {
2044         get_bundle_variant_data(bundle_data, rsc);
2045 
2046         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2047             pe__bundle_replica_t *replica = gIter->data;
2048 
2049             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2050                 passes = TRUE;
2051                 break;
2052             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2053                 passes = TRUE;
2054                 break;
2055             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2056                 passes = TRUE;
2057                 break;
2058             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2059                 passes = TRUE;
2060                 break;
2061             }
2062         }
2063     }
2064 
2065     return !passes;
2066 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
