lib/pengine/bundle.c

/* */
This source file includes following definitions.
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
create_docker_resource
create_podman_resource
create_rkt_resource
disallow_node
create_remote_resource
create_container
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
container_agent_str
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 #include <stdint.h>
  14 
  15 #include <crm/pengine/rules.h>
  16 #include <crm/pengine/status.h>
  17 #include <crm/pengine/internal.h>
  18 #include <crm/msg_xml.h>
  19 #include <crm/common/output.h>
  20 #include <crm/common/xml_internal.h>
  21 #include <pe_status_private.h>
  22 
  23 #define PE__VARIANT_BUNDLE 1
  24 #include "./variant.h"
  25 
  26 static char *
  27 next_ip(const char *last_ip)
     /*  */
  28 {
  29     unsigned int oct1 = 0;
  30     unsigned int oct2 = 0;
  31     unsigned int oct3 = 0;
  32     unsigned int oct4 = 0;
  33     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  34 
  35     if (rc != 4) {
  36         /*@ TODO check for IPv6 */
  37         return NULL;
  38 
  39     } else if (oct3 > 253) {
  40         return NULL;
  41 
  42     } else if (oct4 > 253) {
  43         ++oct3;
  44         oct4 = 1;
  45 
  46     } else {
  47         ++oct4;
  48     }
  49 
  50     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  51 }
  52 
  53 static int
  54 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  55             char *buffer, int max)
  56 {
  57     if(data->ip_range_start == NULL) {
  58         return 0;
  59 
  60     } else if(data->ip_last) {
  61         replica->ipaddr = next_ip(data->ip_last);
  62 
  63     } else {
  64         replica->ipaddr = strdup(data->ip_range_start);
  65     }
  66 
  67     data->ip_last = replica->ipaddr;
  68     switch (data->agent_type) {
  69         case PE__CONTAINER_AGENT_DOCKER:
  70         case PE__CONTAINER_AGENT_PODMAN:
  71             if (data->add_host) {
  72                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
  73                                 data->prefix, replica->offset,
  74                                 replica->ipaddr);
  75             }
  76         case PE__CONTAINER_AGENT_RKT:
  77             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
  78                             replica->ipaddr, data->prefix, replica->offset);
  79         default: // PE__CONTAINER_AGENT_UNKNOWN
  80             break;
  81     }
  82     return 0;
  83 }
  84 
  85 static xmlNode *
  86 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
  87 {
  88     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
  89 
  90     crm_xml_add(rsc, XML_ATTR_ID, name);
  91     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
  92     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
  93     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
  94 
  95     return rsc;
  96 }
  97 
  98 /*!
  99  * \internal
 100  * \brief Check whether cluster can manage resource inside container
 101  *
 102  * \param[in] data  Container variant data
 103  *
 104  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 105  *
 106  * \note The resource is manageable if an IP range or control port has been
 107  *       specified. If a control port is used without an IP range, replicas per
 108  *       host must be 1.
 109  */
 110 static bool
 111 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 112 {
 113     if(data->ip_range_start) {
 114         return TRUE;
 115     }
 116     if(data->control_port) {
 117         if(data->nreplicas_per_host > 1) {
 118             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 119             data->nreplicas_per_host = 1;
 120             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 121         }
 122         return TRUE;
 123     }
 124     return FALSE;
 125 }
 126 
 127 static bool
 128 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 129                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 130 {
 131     if(data->ip_range_start) {
 132         char *id = NULL;
 133         xmlNode *xml_ip = NULL;
 134         xmlNode *xml_obj = NULL;
 135 
 136         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 137         crm_xml_sanitize_id(id);
 138         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 139         free(id);
 140 
 141         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 142         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 143                        data->prefix, replica->offset);
 144 
 145         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 146         if(data->host_network) {
 147             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 148         }
 149 
 150         if(data->host_netmask) {
 151             crm_create_nvpair_xml(xml_obj, NULL,
 152                                   "cidr_netmask", data->host_netmask);
 153 
 154         } else {
 155             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 156         }
 157 
 158         xml_obj = create_xml_node(xml_ip, "operations");
 159         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 160 
 161         // TODO: Other ops? Timeouts and intervals from underlying resource?
 162 
 163         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
 164             return FALSE;
 165         }
 166 
 167         parent->children = g_list_append(parent->children, replica->ip);
 168     }
 169     return TRUE;
 170 }
 171 
 172 static bool
 173 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 174                        pe__bundle_replica_t *replica,
 175                        pe_working_set_t *data_set)
 176 {
 177         int offset = 0, max = 4096;
 178         char *buffer = calloc(1, max+1);
 179 
 180         int doffset = 0, dmax = 1024;
 181         char *dbuffer = calloc(1, dmax+1);
 182 
 183         char *id = NULL;
 184         xmlNode *xml_container = NULL;
 185         xmlNode *xml_obj = NULL;
 186 
 187         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
 188         crm_xml_sanitize_id(id);
 189         xml_container = create_resource(id, "heartbeat",
 190                                         PE__CONTAINER_AGENT_DOCKER_S);
 191         free(id);
 192 
 193         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 194         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 195                        data->prefix, replica->offset);
 196 
 197         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 198         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 199         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 200         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 201 
 202         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 203 
 204         /* Set a container hostname only if we have an IP to map it to.
 205          * The user can set -h or --uts=host themselves if they want a nicer
 206          * name for logs, but this makes applications happy who need their
 207          * hostname to match the IP they bind to.
 208          */
 209         if (data->ip_range_start != NULL) {
 210             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 211                                data->prefix, replica->offset);
 212         }
 213 
 214         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 215 
 216         if (data->container_network) {
 217 #if 0
 218             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 219                                replica->ipaddr);
 220 #endif
 221             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 222                                data->container_network);
 223         }
 224 
 225         if(data->control_port) {
 226             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 227         } else {
 228             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 229         }
 230 
 231         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 232             pe__bundle_mount_t *mount = pIter->data;
 233 
 234             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 235                 char *source = crm_strdup_printf(
 236                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 237 
 238                 if(doffset > 0) {
 239                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 240                 }
 241                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 242                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 243                 free(source);
 244 
 245             } else {
 246                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 247             }
 248             if(mount->options) {
 249                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 250             }
 251         }
 252 
 253         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 254             pe__bundle_port_t *port = pIter->data;
 255 
 256             if (replica->ipaddr) {
 257                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 258                                    replica->ipaddr, port->source,
 259                                    port->target);
 260             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 261                 // No need to do port mapping if net=host
 262                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 263             }
 264         }
 265 
 266         if (data->launcher_options) {
 267             offset += snprintf(buffer+offset, max-offset, " %s",
 268                                data->launcher_options);
 269         }
 270 
 271         if (data->container_host_options) {
 272             offset += snprintf(buffer + offset, max - offset, " %s",
 273                                data->container_host_options);
 274         }
 275 
 276         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 277         free(buffer);
 278 
 279         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 280         free(dbuffer);
 281 
 282         if (replica->child) {
 283             if (data->container_command) {
 284                 crm_create_nvpair_xml(xml_obj, NULL,
 285                                       "run_cmd", data->container_command);
 286             } else {
 287                 crm_create_nvpair_xml(xml_obj, NULL,
 288                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 289             }
 290 
 291             /* TODO: Allow users to specify their own?
 292              *
 293              * We just want to know if the container is alive, we'll
 294              * monitor the child independently
 295              */
 296             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 297 #if 0
 298         /* @TODO Consider supporting the use case where we can start and stop
 299          * resources, but not proxy local commands (such as setting node
 300          * attributes), by running the local executor in stand-alone mode.
 301          * However, this would probably be better done via ACLs as with other
 302          * Pacemaker Remote nodes.
 303          */
 304         } else if ((child != NULL) && data->untrusted) {
 305             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 306                                   CRM_DAEMON_DIR "/pacemaker-execd");
 307             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 308                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 309 #endif
 310         } else {
 311             if (data->container_command) {
 312                 crm_create_nvpair_xml(xml_obj, NULL,
 313                                       "run_cmd", data->container_command);
 314             }
 315 
 316             /* TODO: Allow users to specify their own?
 317              *
 318              * We don't know what's in the container, so we just want
 319              * to know if it is alive
 320              */
 321             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 322         }
 323 
 324 
 325         xml_obj = create_xml_node(xml_container, "operations");
 326         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 327 
 328         // TODO: Other ops? Timeouts and intervals from underlying resource?
 329         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 330             return FALSE;
 331         }
 332         parent->children = g_list_append(parent->children, replica->container);
 333         return TRUE;
 334 }
 335 
 336 static bool
 337 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 338                        pe__bundle_replica_t *replica,
 339                        pe_working_set_t *data_set)
 340 {
 341         int offset = 0, max = 4096;
 342         char *buffer = calloc(1, max+1);
 343 
 344         int doffset = 0, dmax = 1024;
 345         char *dbuffer = calloc(1, dmax+1);
 346 
 347         char *id = NULL;
 348         xmlNode *xml_container = NULL;
 349         xmlNode *xml_obj = NULL;
 350 
 351         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
 352         crm_xml_sanitize_id(id);
 353         xml_container = create_resource(id, "heartbeat",
 354                                         PE__CONTAINER_AGENT_PODMAN_S);
 355         free(id);
 356 
 357         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 358         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 359                        data->prefix, replica->offset);
 360 
 361         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 362         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 363         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 364         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 365 
 366         // FIXME: (bandini 2018-08) podman has no restart policies
 367         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 368 
 369         /* Set a container hostname only if we have an IP to map it to.
 370          * The user can set -h or --uts=host themselves if they want a nicer
 371          * name for logs, but this makes applications happy who need their
 372          * hostname to match the IP they bind to.
 373          */
 374         if (data->ip_range_start != NULL) {
 375             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 376                                data->prefix, replica->offset);
 377         }
 378 
 379         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 380 
 381         if (data->container_network) {
 382 #if 0
 383             // podman has no support for --link-local-ip
 384             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 385                                replica->ipaddr);
 386 #endif
 387             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 388                                data->container_network);
 389         }
 390 
 391         if(data->control_port) {
 392             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 393         } else {
 394             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 395         }
 396 
 397         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 398             pe__bundle_mount_t *mount = pIter->data;
 399 
 400             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 401                 char *source = crm_strdup_printf(
 402                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 403 
 404                 if(doffset > 0) {
 405                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 406                 }
 407                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 408                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 409                 free(source);
 410 
 411             } else {
 412                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 413             }
 414             if(mount->options) {
 415                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 416             }
 417         }
 418 
 419         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 420             pe__bundle_port_t *port = pIter->data;
 421 
 422             if (replica->ipaddr) {
 423                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 424                                    replica->ipaddr, port->source,
 425                                    port->target);
 426             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 427                 // No need to do port mapping if net=host
 428                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 429             }
 430         }
 431 
 432         if (data->launcher_options) {
 433             offset += snprintf(buffer+offset, max-offset, " %s",
 434                                data->launcher_options);
 435         }
 436 
 437         if (data->container_host_options) {
 438             offset += snprintf(buffer + offset, max - offset, " %s",
 439                                data->container_host_options);
 440         }
 441 
 442         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 443         free(buffer);
 444 
 445         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 446         free(dbuffer);
 447 
 448         if (replica->child) {
 449             if (data->container_command) {
 450                 crm_create_nvpair_xml(xml_obj, NULL,
 451                                       "run_cmd", data->container_command);
 452             } else {
 453                 crm_create_nvpair_xml(xml_obj, NULL,
 454                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 455             }
 456 
 457             /* TODO: Allow users to specify their own?
 458              *
 459              * We just want to know if the container is alive, we'll
 460              * monitor the child independently
 461              */
 462             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 463 #if 0
 464         /* @TODO Consider supporting the use case where we can start and stop
 465          * resources, but not proxy local commands (such as setting node
 466          * attributes), by running the local executor in stand-alone mode.
 467          * However, this would probably be better done via ACLs as with other
 468          * Pacemaker Remote nodes.
 469          */
 470         } else if ((child != NULL) && data->untrusted) {
 471             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 472                                   CRM_DAEMON_DIR "/pacemaker-execd");
 473             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 474                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 475 #endif
 476         } else {
 477             if (data->container_command) {
 478                 crm_create_nvpair_xml(xml_obj, NULL,
 479                                       "run_cmd", data->container_command);
 480             }
 481 
 482             /* TODO: Allow users to specify their own?
 483              *
 484              * We don't know what's in the container, so we just want
 485              * to know if it is alive
 486              */
 487             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 488         }
 489 
 490 
 491         xml_obj = create_xml_node(xml_container, "operations");
 492         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 493 
 494         // TODO: Other ops? Timeouts and intervals from underlying resource?
 495         if (!common_unpack(xml_container, &replica->container, parent,
 496                            data_set)) {
 497             return FALSE;
 498         }
 499         parent->children = g_list_append(parent->children, replica->container);
 500         return TRUE;
 501 }
 502 
 503 static bool
 504 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 505                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 506 {
 507         int offset = 0, max = 4096;
 508         char *buffer = calloc(1, max+1);
 509 
 510         int doffset = 0, dmax = 1024;
 511         char *dbuffer = calloc(1, dmax+1);
 512 
 513         char *id = NULL;
 514         xmlNode *xml_container = NULL;
 515         xmlNode *xml_obj = NULL;
 516 
 517         int volid = 0;
 518 
 519         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
 520         crm_xml_sanitize_id(id);
 521         xml_container = create_resource(id, "heartbeat",
 522                                         PE__CONTAINER_AGENT_RKT_S);
 523         free(id);
 524 
 525         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 526         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 527                        data->prefix, replica->offset);
 528 
 529         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 530         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
 531         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
 532         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 533 
 534         /* Set a container hostname only if we have an IP to map it to.
 535          * The user can set -h or --uts=host themselves if they want a nicer
 536          * name for logs, but this makes applications happy who need their
 537          * hostname to match the IP they bind to.
 538          */
 539         if (data->ip_range_start != NULL) {
 540             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
 541                                data->prefix, replica->offset);
 542         }
 543 
 544         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 545 
 546         if (data->container_network) {
 547 #if 0
 548             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 549                                replica->ipaddr);
 550 #endif
 551             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 552                                data->container_network);
 553         }
 554 
 555         if(data->control_port) {
 556             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
 557         } else {
 558             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 559         }
 560 
 561         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 562             pe__bundle_mount_t *mount = pIter->data;
 563 
 564             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 565                 char *source = crm_strdup_printf(
 566                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 567 
 568                 if(doffset > 0) {
 569                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 570                 }
 571                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 572                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
 573                 if(mount->options) {
 574                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 575                 }
 576                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 577                 free(source);
 578 
 579             } else {
 580                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
 581                 if(mount->options) {
 582                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 583                 }
 584                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 585             }
 586             volid++;
 587         }
 588 
 589         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 590             pe__bundle_port_t *port = pIter->data;
 591 
 592             if (replica->ipaddr) {
 593                 offset += snprintf(buffer+offset, max-offset,
 594                                    " --port=%s:%s:%s", port->target,
 595                                    replica->ipaddr, port->source);
 596             } else {
 597                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
 598             }
 599         }
 600 
 601         if (data->launcher_options) {
 602             offset += snprintf(buffer+offset, max-offset, " %s",
 603                                data->launcher_options);
 604         }
 605 
 606         if (data->container_host_options) {
 607             offset += snprintf(buffer + offset, max - offset, " %s",
 608                                data->container_host_options);
 609         }
 610 
 611         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 612         free(buffer);
 613 
 614         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 615         free(dbuffer);
 616 
 617         if (replica->child) {
 618             if (data->container_command) {
 619                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 620                                       data->container_command);
 621             } else {
 622                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 623                                       SBIN_DIR "/pacemaker-remoted");
 624             }
 625 
 626             /* TODO: Allow users to specify their own?
 627              *
 628              * We just want to know if the container is alive, we'll
 629              * monitor the child independently
 630              */
 631             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 632 #if 0
 633         /* @TODO Consider supporting the use case where we can start and stop
 634          * resources, but not proxy local commands (such as setting node
 635          * attributes), by running the local executor in stand-alone mode.
 636          * However, this would probably be better done via ACLs as with other
 637          * Pacemaker Remote nodes.
 638          */
 639         } else if ((child != NULL) && data->untrusted) {
 640             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 641                                   CRM_DAEMON_DIR "/pacemaker-execd");
 642             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 643                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 644 #endif
 645         } else {
 646             if (data->container_command) {
 647                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 648                                       data->container_command);
 649             }
 650 
 651             /* TODO: Allow users to specify their own?
 652              *
 653              * We don't know what's in the container, so we just want
 654              * to know if it is alive
 655              */
 656             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 657         }
 658 
 659 
 660         xml_obj = create_xml_node(xml_container, "operations");
 661         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 662 
 663         // TODO: Other ops? Timeouts and intervals from underlying resource?
 664 
 665         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 666             return FALSE;
 667         }
 668         parent->children = g_list_append(parent->children, replica->container);
 669         return TRUE;
 670 }
 671 
 672 /*!
 673  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 674  *
 675  * \param[in,out] rsc    Resource to modify
 676  * \param[in]     uname  Name of node to ban
 677  */
 678 static void
 679 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 680 {
 681     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 682 
 683     if (match) {
 684         ((pe_node_t *) match)->weight = -INFINITY;
 685         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 686     }
 687     if (rsc->children) {
 688         g_list_foreach(rsc->children, (GFunc) disallow_node, (gpointer) uname);
 689     }
 690 }
 691 
 692 static bool
 693 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 694                        pe__bundle_replica_t *replica,
 695                        pe_working_set_t *data_set)
 696 {
 697     if (replica->child && valid_network(data)) {
 698         GHashTableIter gIter;
 699         pe_node_t *node = NULL;
 700         xmlNode *xml_remote = NULL;
 701         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 702         char *port_s = NULL;
 703         const char *uname = NULL;
 704         const char *connect_name = NULL;
 705 
 706         if (pe_find_resource(data_set->resources, id) != NULL) {
 707             free(id);
 708             // The biggest hammer we have
 709             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 710                                    replica->child->id, replica->offset);
 711             //@TODO return false instead of asserting?
 712             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
 713         }
 714 
 715         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 716          * connection does not have its own IP is a magic string that we use to
 717          * support nested remotes (i.e. a bundle running on a remote node).
 718          */
 719         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 720 
 721         if (data->control_port == NULL) {
 722             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 723         }
 724 
 725         /* This sets replica->container as replica->remote's container, which is
 726          * similar to what happens with guest nodes. This is how the scheduler
 727          * knows that the bundle node is fenced by recovering the container, and
 728          * that remote should be ordered relative to the container.
 729          */
 730         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 731                                           NULL, NULL, NULL,
 732                                           connect_name, (data->control_port?
 733                                           data->control_port : port_s));
 734         free(port_s);
 735 
 736         /* Abandon our created ID, and pull the copy from the XML, because we
 737          * need something that will get freed during data set cleanup to use as
 738          * the node ID and uname.
 739          */
 740         free(id);
 741         id = NULL;
 742         uname = ID(xml_remote);
 743 
 744         /* Ensure a node has been created for the guest (it may have already
 745          * been, if it has a permanent node attribute), and ensure its weight is
 746          * -INFINITY so no other resources can run on it.
 747          */
 748         node = pe_find_node(data_set->nodes, uname);
 749         if (node == NULL) {
 750             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 751                                   data_set);
 752         } else {
 753             node->weight = -INFINITY;
 754         }
 755         node->rsc_discover_mode = pe_discover_never;
 756 
 757         /* unpack_remote_nodes() ensures that each remote node and guest node
 758          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 759          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 760          * what nodes will be needed, so we do it just above.
 761          *
 762          * Worse, that means that the node may have been utilized while
 763          * unpacking other resources, without our weight correction. The most
 764          * likely place for this to happen is when common_unpack() calls
 765          * resource_location() to set a default score in symmetric clusters.
 766          * This adds a node *copy* to each resource's allowed nodes, and these
 767          * copies will have the wrong weight.
 768          *
 769          * As a hacky workaround, fix those copies here.
 770          *
 771          * @TODO Possible alternative: ensure bundles are unpacked before other
 772          * resources, so the weight is correct before any copies are made.
 773          */
 774         g_list_foreach(data_set->resources, (GFunc) disallow_node, (gpointer) uname);
 775 
 776         replica->node = pe__copy_node(node);
 777         replica->node->weight = 500;
 778         replica->node->rsc_discover_mode = pe_discover_exclusive;
 779 
 780         /* Ensure the node shows up as allowed and with the correct discovery set */
 781         if (replica->child->allowed_nodes != NULL) {
 782             g_hash_table_destroy(replica->child->allowed_nodes);
 783         }
 784         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 785         g_hash_table_insert(replica->child->allowed_nodes,
 786                             (gpointer) replica->node->details->id,
 787                             pe__copy_node(replica->node));
 788 
 789         {
 790             pe_node_t *copy = pe__copy_node(replica->node);
 791             copy->weight = -INFINITY;
 792             g_hash_table_insert(replica->child->parent->allowed_nodes,
 793                                 (gpointer) replica->node->details->id, copy);
 794         }
 795         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
 796             return FALSE;
 797         }
 798 
 799         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 800         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 801             if (pe__is_guest_or_remote_node(node)) {
 802                 /* Remote resources can only run on 'normal' cluster node */
 803                 node->weight = -INFINITY;
 804             }
 805         }
 806 
 807         replica->node->details->remote_rsc = replica->remote;
 808 
 809         // Ensure pe__is_guest_node() functions correctly immediately
 810         replica->remote->container = replica->container;
 811 
 812         /* A bundle's #kind is closer to "container" (guest node) than the
 813          * "remote" set by pe_create_node().
 814          */
 815         g_hash_table_insert(replica->node->details->attrs,
 816                             strdup(CRM_ATTR_KIND), strdup("container"));
 817 
 818         /* One effect of this is that setup_container() will add
 819          * replica->remote to replica->container's fillers, which will make
 820          * pe__resource_contains_guest_node() true for replica->container.
 821          *
 822          * replica->child does NOT get added to replica->container's fillers.
 823          * The only noticeable effect if it did would be for its fail count to
 824          * be taken into account when checking replica->container's migration
 825          * threshold.
 826          */
 827         parent->children = g_list_append(parent->children, replica->remote);
 828     }
 829     return TRUE;
 830 }
 831 
 832 static bool
 833 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 834                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 835 {
 836 
 837     switch (data->agent_type) {
 838         case PE__CONTAINER_AGENT_DOCKER:
 839             if (!create_docker_resource(parent, data, replica, data_set)) {
 840                 return FALSE;
 841             }
 842             break;
 843 
 844         case PE__CONTAINER_AGENT_PODMAN:
 845             if (!create_podman_resource(parent, data, replica, data_set)) {
 846                 return FALSE;
 847             }
 848             break;
 849 
 850         case PE__CONTAINER_AGENT_RKT:
 851             if (!create_rkt_resource(parent, data, replica, data_set)) {
 852                 return FALSE;
 853             }
 854             break;
 855         default: // PE__CONTAINER_AGENT_UNKNOWN
 856             return FALSE;
 857     }
 858 
 859     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
 860         return FALSE;
 861     }
 862     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
 863         return FALSE;
 864     }
 865     if (replica->child && replica->ipaddr) {
 866         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 867     }
 868 
 869     if (replica->remote) {
 870         /*
 871          * Allow the remote connection resource to be allocated to a
 872          * different node than the one on which the container is active.
 873          *
 874          * This makes it possible to have Pacemaker Remote nodes running
 875          * containers with pacemaker-remoted inside in order to start
 876          * services inside those containers.
 877          */
 878         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 879     }
 880 
 881     return TRUE;
 882 }
 883 
 884 static void
 885 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 886           const char *target, const char *options, uint32_t flags)
 887 {
 888     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 889 
 890     CRM_ASSERT(mount != NULL);
 891     mount->source = strdup(source);
 892     mount->target = strdup(target);
 893     pcmk__str_update(&mount->options, options);
 894     mount->flags = flags;
 895     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 896 }
 897 
 898 static void
 899 mount_free(pe__bundle_mount_t *mount)
     /*  */
 900 {
 901     free(mount->source);
 902     free(mount->target);
 903     free(mount->options);
 904     free(mount);
 905 }
 906 
 907 static void
 908 port_free(pe__bundle_port_t *port)
     /*  */
 909 {
 910     free(port->source);
 911     free(port->target);
 912     free(port);
 913 }
 914 
 915 static pe__bundle_replica_t *
 916 replica_for_remote(pe_resource_t *remote)
     /*  */
 917 {
 918     pe_resource_t *top = remote;
 919     pe__bundle_variant_data_t *bundle_data = NULL;
 920 
 921     if (top == NULL) {
 922         return NULL;
 923     }
 924 
 925     while (top->parent != NULL) {
 926         top = top->parent;
 927     }
 928 
 929     get_bundle_variant_data(bundle_data, top);
 930     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 931          gIter = gIter->next) {
 932         pe__bundle_replica_t *replica = gIter->data;
 933 
 934         if (replica->remote == remote) {
 935             return replica;
 936         }
 937     }
 938     CRM_LOG_ASSERT(FALSE);
 939     return NULL;
 940 }
 941 
 942 bool
 943 pe__bundle_needs_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 944 {
 945     const char *value;
 946     GHashTable *params = NULL;
 947 
 948     if (rsc == NULL) {
 949         return false;
 950     }
 951 
 952     // Use NULL node since pcmk__bundle_expand() uses that to set value
 953     params = pe_rsc_params(rsc, NULL, data_set);
 954     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 955 
 956     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 957            && xml_contains_remote_node(rsc->xml);
 958 }
 959 
 960 const char *
 961 pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
     /*  */
 962                            xmlNode *xml, const char *field)
 963 {
 964     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 965 
 966     pe_node_t *node = NULL;
 967     pe__bundle_replica_t *replica = NULL;
 968 
 969     if (!pe__bundle_needs_remote_name(rsc, data_set)) {
 970         return NULL;
 971     }
 972 
 973     replica = replica_for_remote(rsc);
 974     if (replica == NULL) {
 975         return NULL;
 976     }
 977 
 978     node = replica->container->allocated_to;
 979     if (node == NULL) {
 980         /* If it won't be running anywhere after the
 981          * transition, go with where it's running now.
 982          */
 983         node = pe__current_node(replica->container);
 984     }
 985 
 986     if(node == NULL) {
 987         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 988         return NULL;
 989     }
 990 
 991     crm_trace("Setting address for bundle connection %s to bundle host %s",
 992               rsc->id, node->details->uname);
 993     if(xml != NULL && field != NULL) {
 994         crm_xml_add(xml, field, node->details->uname);
 995     }
 996 
 997     return node->details->uname;
 998 }
 999 
1000 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
1001         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
1002                                    "Bundle mount", ID(mount_xml), flags,    \
1003                                    (flags_to_set), #flags_to_set);          \
1004     } while (0)
1005 
1006 gboolean
1007 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
1008 {
1009     const char *value = NULL;
1010     xmlNode *xml_obj = NULL;
1011     xmlNode *xml_resource = NULL;
1012     pe__bundle_variant_data_t *bundle_data = NULL;
1013     bool need_log_mount = TRUE;
1014 
1015     CRM_ASSERT(rsc != NULL);
1016     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
1017 
1018     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
1019     rsc->variant_opaque = bundle_data;
1020     bundle_data->prefix = strdup(rsc->id);
1021 
1022     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1023     if (xml_obj != NULL) {
1024         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1025     } else {
1026         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1027         if (xml_obj != NULL) {
1028             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1029         } else {
1030             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1031             if (xml_obj != NULL) {
1032                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1033             } else {
1034                 return FALSE;
1035             }
1036         }
1037     }
1038 
1039     // Use 0 for default, minimum, and invalid promoted-max
1040     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
1041     if (value == NULL) {
1042         // @COMPAT deprecated since 2.0.0
1043         value = crm_element_value(xml_obj, "masters");
1044     }
1045     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1046 
1047     // Default replicas to promoted-max if it was specified and 1 otherwise
1048     value = crm_element_value(xml_obj, "replicas");
1049     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1050         bundle_data->nreplicas = bundle_data->promoted_max;
1051     } else {
1052         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1053     }
1054 
1055     /*
1056      * Communication between containers on the same host via the
1057      * floating IPs only works if the container is started with:
1058      *   --userland-proxy=false --ip-masq=false
1059      */
1060     value = crm_element_value(xml_obj, "replicas-per-host");
1061     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1062     if (bundle_data->nreplicas_per_host == 1) {
1063         pe__clear_resource_flags(rsc, pe_rsc_unique);
1064     }
1065 
1066     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1067     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1068     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1069     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1070 
1071     xml_obj = first_named_child(rsc->xml, "network");
1072     if(xml_obj) {
1073 
1074         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1075         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1076         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1077         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1078         value = crm_element_value(xml_obj, "add-host");
1079         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1080             bundle_data->add_host = TRUE;
1081         }
1082 
1083         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1084              xml_child = pcmk__xe_next(xml_child)) {
1085 
1086             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1087             port->source = crm_element_value_copy(xml_child, "port");
1088 
1089             if(port->source == NULL) {
1090                 port->source = crm_element_value_copy(xml_child, "range");
1091             } else {
1092                 port->target = crm_element_value_copy(xml_child, "internal-port");
1093             }
1094 
1095             if(port->source != NULL && strlen(port->source) > 0) {
1096                 if(port->target == NULL) {
1097                     port->target = strdup(port->source);
1098                 }
1099                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1100 
1101             } else {
1102                 pe_err("Invalid port directive %s", ID(xml_child));
1103                 port_free(port);
1104             }
1105         }
1106     }
1107 
1108     xml_obj = first_named_child(rsc->xml, "storage");
1109     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1110          xml_child = pcmk__xe_next(xml_child)) {
1111 
1112         const char *source = crm_element_value(xml_child, "source-dir");
1113         const char *target = crm_element_value(xml_child, "target-dir");
1114         const char *options = crm_element_value(xml_child, "options");
1115         int flags = pe__bundle_mount_none;
1116 
1117         if (source == NULL) {
1118             source = crm_element_value(xml_child, "source-dir-root");
1119             pe__set_bundle_mount_flags(xml_child, flags,
1120                                        pe__bundle_mount_subdir);
1121         }
1122 
1123         if (source && target) {
1124             mount_add(bundle_data, source, target, options, flags);
1125             if (strcmp(target, "/var/log") == 0) {
1126                 need_log_mount = FALSE;
1127             }
1128         } else {
1129             pe_err("Invalid mount directive %s", ID(xml_child));
1130         }
1131     }
1132 
1133     xml_obj = first_named_child(rsc->xml, "primitive");
1134     if (xml_obj && valid_network(bundle_data)) {
1135         char *value = NULL;
1136         xmlNode *xml_set = NULL;
1137 
1138         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1139 
1140         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1141          * part of the resource name, so that bundles don't restart in a rolling
1142          * upgrade. (It also avoids needing to change regression tests.)
1143          */
1144         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1145                       (bundle_data->promoted_max? "master"
1146                       : (const char *)xml_resource->name));
1147 
1148         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1149         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1150 
1151         crm_create_nvpair_xml(xml_set, NULL,
1152                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1153 
1154         value = pcmk__itoa(bundle_data->nreplicas);
1155         crm_create_nvpair_xml(xml_set, NULL,
1156                               XML_RSC_ATTR_INCARNATION_MAX, value);
1157         free(value);
1158 
1159         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1160         crm_create_nvpair_xml(xml_set, NULL,
1161                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
1162         free(value);
1163 
1164         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1165                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1166 
1167         if (bundle_data->promoted_max) {
1168             crm_create_nvpair_xml(xml_set, NULL,
1169                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1170 
1171             value = pcmk__itoa(bundle_data->promoted_max);
1172             crm_create_nvpair_xml(xml_set, NULL,
1173                                   XML_RSC_ATTR_PROMOTED_MAX, value);
1174             free(value);
1175         }
1176 
1177         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1178         add_node_copy(xml_resource, xml_obj);
1179 
1180     } else if(xml_obj) {
1181         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1182                rsc->id, ID(xml_obj));
1183         return FALSE;
1184     }
1185 
1186     if(xml_resource) {
1187         int lpc = 0;
1188         GList *childIter = NULL;
1189         pe_resource_t *new_rsc = NULL;
1190         pe__bundle_port_t *port = NULL;
1191 
1192         int offset = 0, max = 1024;
1193         char *buffer = NULL;
1194 
1195         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
1196             pe_err("Failed unpacking resource %s", ID(rsc->xml));
1197             if (new_rsc != NULL && new_rsc->fns != NULL) {
1198                 new_rsc->fns->free(new_rsc);
1199             }
1200             return FALSE;
1201         }
1202 
1203         bundle_data->child = new_rsc;
1204 
1205         /* Currently, we always map the default authentication key location
1206          * into the same location inside the container.
1207          *
1208          * Ideally, we would respect the host's PCMK_authkey_location, but:
1209          * - it may be different on different nodes;
1210          * - the actual connection will do extra checking to make sure the key
1211          *   file exists and is readable, that we can't do here on the DC
1212          * - tools such as crm_resource and crm_simulate may not have the same
1213          *   environment variables as the cluster, causing operation digests to
1214          *   differ
1215          *
1216          * Always using the default location inside the container is fine,
1217          * because we control the pacemaker_remote environment, and it avoids
1218          * having to pass another environment variable to the container.
1219          *
1220          * @TODO A better solution may be to have only pacemaker_remote use the
1221          * environment variable, and have the cluster nodes use a new
1222          * cluster option for key location. This would introduce the limitation
1223          * of the location being the same on all cluster nodes, but that's
1224          * reasonable.
1225          */
1226         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1227                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1228 
1229         if (need_log_mount) {
1230             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1231                       pe__bundle_mount_subdir);
1232         }
1233 
1234         port = calloc(1, sizeof(pe__bundle_port_t));
1235         if(bundle_data->control_port) {
1236             port->source = strdup(bundle_data->control_port);
1237         } else {
1238             /* If we wanted to respect PCMK_remote_port, we could use
1239              * crm_default_remote_port() here and elsewhere in this file instead
1240              * of DEFAULT_REMOTE_PORT.
1241              *
1242              * However, it gains nothing, since we control both the container
1243              * environment and the connection resource parameters, and the user
1244              * can use a different port if desired by setting control-port.
1245              */
1246             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1247         }
1248         port->target = strdup(port->source);
1249         bundle_data->ports = g_list_append(bundle_data->ports, port);
1250 
1251         buffer = calloc(1, max+1);
1252         for (childIter = bundle_data->child->children; childIter != NULL;
1253              childIter = childIter->next) {
1254 
1255             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1256 
1257             replica->child = childIter->data;
1258             replica->child->exclusive_discover = TRUE;
1259             replica->offset = lpc++;
1260 
1261             // Ensure the child's notify gets set based on the underlying primitive's value
1262             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1263                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1264             }
1265 
1266             offset += allocate_ip(bundle_data, replica, buffer+offset,
1267                                   max-offset);
1268             bundle_data->replicas = g_list_append(bundle_data->replicas,
1269                                                   replica);
1270             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1271                                                                 XML_RSC_ATTR_TARGET);
1272         }
1273         bundle_data->container_host_options = buffer;
1274         if (bundle_data->attribute_target) {
1275             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1276                                  strdup(bundle_data->attribute_target));
1277             g_hash_table_replace(bundle_data->child->meta,
1278                                  strdup(XML_RSC_ATTR_TARGET),
1279                                  strdup(bundle_data->attribute_target));
1280         }
1281 
1282     } else {
1283         // Just a naked container, no pacemaker-remote
1284         int offset = 0, max = 1024;
1285         char *buffer = calloc(1, max+1);
1286 
1287         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1288             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1289 
1290             replica->offset = lpc;
1291             offset += allocate_ip(bundle_data, replica, buffer+offset,
1292                                   max-offset);
1293             bundle_data->replicas = g_list_append(bundle_data->replicas,
1294                                                   replica);
1295         }
1296         bundle_data->container_host_options = buffer;
1297     }
1298 
1299     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1300          gIter = gIter->next) {
1301         pe__bundle_replica_t *replica = gIter->data;
1302 
1303         if (!create_container(rsc, bundle_data, replica, data_set)) {
1304             pe_err("Failed unpacking resource %s", rsc->id);
1305             rsc->fns->free(rsc);
1306             return FALSE;
1307         }
1308 
1309         /* Utilization needs special handling for bundles. It makes no sense for
1310          * the inner primitive to have utilization, because it is tied
1311          * one-to-one to the guest node created by the container resource -- and
1312          * there's no way to set capacities for that guest node anyway.
1313          *
1314          * What the user really wants is to configure utilization for the
1315          * container. However, the schema only allows utilization for
1316          * primitives, and the container resource is implicit anyway, so the
1317          * user can *only* configure utilization for the inner primitive. If
1318          * they do, move the primitive's utilization values to the container.
1319          *
1320          * @TODO This means that bundles without an inner primitive can't have
1321          * utilization. An alternative might be to allow utilization values in
1322          * the top-level bundle XML in the schema, and copy those to each
1323          * container.
1324          */
1325         if (replica->child != NULL) {
1326             GHashTable *empty = replica->container->utilization;
1327 
1328             replica->container->utilization = replica->child->utilization;
1329             replica->child->utilization = empty;
1330         }
1331     }
1332 
1333     if (bundle_data->child) {
1334         rsc->children = g_list_append(rsc->children, bundle_data->child);
1335     }
1336     return TRUE;
1337 }
1338 
1339 static int
1340 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1341 {
1342     if (rsc) {
1343         gboolean child_active = rsc->fns->active(rsc, all);
1344 
1345         if (child_active && !all) {
1346             return TRUE;
1347         } else if (!child_active && all) {
1348             return FALSE;
1349         }
1350     }
1351     return -1;
1352 }
1353 
1354 gboolean
1355 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1356 {
1357     pe__bundle_variant_data_t *bundle_data = NULL;
1358     GList *iter = NULL;
1359 
1360     get_bundle_variant_data(bundle_data, rsc);
1361     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1362         pe__bundle_replica_t *replica = iter->data;
1363         int rsc_active;
1364 
1365         rsc_active = replica_resource_active(replica->ip, all);
1366         if (rsc_active >= 0) {
1367             return (gboolean) rsc_active;
1368         }
1369 
1370         rsc_active = replica_resource_active(replica->child, all);
1371         if (rsc_active >= 0) {
1372             return (gboolean) rsc_active;
1373         }
1374 
1375         rsc_active = replica_resource_active(replica->container, all);
1376         if (rsc_active >= 0) {
1377             return (gboolean) rsc_active;
1378         }
1379 
1380         rsc_active = replica_resource_active(replica->remote, all);
1381         if (rsc_active >= 0) {
1382             return (gboolean) rsc_active;
1383         }
1384     }
1385 
1386     /* If "all" is TRUE, we've already checked that no resources were inactive,
1387      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1388      * so return FALSE.
1389      */
1390     return all;
1391 }
1392 
1393 /*!
1394  * \internal
1395  * \brief Find the bundle replica corresponding to a given node
1396  *
1397  * \param[in] bundle  Top-level bundle resource
1398  * \param[in] node    Node to search for
1399  *
1400  * \return Bundle replica if found, NULL otherwise
1401  */
1402 pe_resource_t *
1403 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1404 {
1405     pe__bundle_variant_data_t *bundle_data = NULL;
1406     CRM_ASSERT(bundle && node);
1407 
1408     get_bundle_variant_data(bundle_data, bundle);
1409     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1410          gIter = gIter->next) {
1411         pe__bundle_replica_t *replica = gIter->data;
1412 
1413         CRM_ASSERT(replica && replica->node);
1414         if (replica->node->details == node->details) {
1415             return replica->child;
1416         }
1417     }
1418     return NULL;
1419 }
1420 
1421 static void
1422 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1423                   void *print_data)
1424 {
1425     if (rsc != NULL) {
1426         if (options & pe_print_html) {
1427             status_print("<li>");
1428         }
1429         rsc->fns->print(rsc, pre_text, options, print_data);
1430         if (options & pe_print_html) {
1431             status_print("</li>\n");
1432         }
1433     }
1434 }
1435 
1436 static const char*
1437 container_agent_str(enum pe__container_agent t)
     /*  */
1438 {
1439     switch (t) {
1440         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
1441         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
1442         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
1443         default: // PE__CONTAINER_AGENT_UNKNOWN
1444             break;
1445     }
1446     return PE__CONTAINER_AGENT_UNKNOWN_S;
1447 }
1448 
1449 static void
1450 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1451                  void *print_data)
1452 {
1453     pe__bundle_variant_data_t *bundle_data = NULL;
1454     char *child_text = NULL;
1455     CRM_CHECK(rsc != NULL, return);
1456 
1457     if (pre_text == NULL) {
1458         pre_text = "";
1459     }
1460     child_text = crm_strdup_printf("%s        ", pre_text);
1461 
1462     get_bundle_variant_data(bundle_data, rsc);
1463 
1464     status_print("%s<bundle ", pre_text);
1465     status_print("id=\"%s\" ", rsc->id);
1466     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1467     status_print("image=\"%s\" ", bundle_data->image);
1468     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1469     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1470     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1471     status_print(">\n");
1472 
1473     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1474          gIter = gIter->next) {
1475         pe__bundle_replica_t *replica = gIter->data;
1476 
1477         CRM_ASSERT(replica);
1478         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
1479         print_rsc_in_list(replica->ip, child_text, options, print_data);
1480         print_rsc_in_list(replica->child, child_text, options, print_data);
1481         print_rsc_in_list(replica->container, child_text, options, print_data);
1482         print_rsc_in_list(replica->remote, child_text, options, print_data);
1483         status_print("%s    </replica>\n", pre_text);
1484     }
1485     status_print("%s</bundle>\n", pre_text);
1486     free(child_text);
1487 }
1488 
1489 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1490 int
1491 pe__bundle_xml(pcmk__output_t *out, va_list args)
1492 {
1493     uint32_t show_opts = va_arg(args, uint32_t);
1494     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1495     GList *only_node = va_arg(args, GList *);
1496     GList *only_rsc = va_arg(args, GList *);
1497 
1498     pe__bundle_variant_data_t *bundle_data = NULL;
1499     int rc = pcmk_rc_no_output;
1500     gboolean printed_header = FALSE;
1501     gboolean print_everything = TRUE;
1502 
1503     CRM_ASSERT(rsc != NULL);
1504 
1505     get_bundle_variant_data(bundle_data, rsc);
1506 
1507     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1508         return rc;
1509     }
1510 
1511     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1512 
1513     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1514          gIter = gIter->next) {
1515         pe__bundle_replica_t *replica = gIter->data;
1516         char *id = NULL;
1517         gboolean print_ip, print_child, print_ctnr, print_remote;
1518 
1519         CRM_ASSERT(replica);
1520 
1521         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1522             continue;
1523         }
1524 
1525         print_ip = replica->ip != NULL &&
1526                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1527         print_child = replica->child != NULL &&
1528                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1529         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1530         print_remote = replica->remote != NULL &&
1531                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1532 
1533         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1534             continue;
1535         }
1536 
1537         if (!printed_header) {
1538             printed_header = TRUE;
1539 
1540             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6,
1541                      "id", rsc->id,
1542                      "type", container_agent_str(bundle_data->agent_type),
1543                      "image", bundle_data->image,
1544                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1545                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1546                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed));
1547             CRM_ASSERT(rc == pcmk_rc_ok);
1548         }
1549 
1550         id = pcmk__itoa(replica->offset);
1551         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1552         free(id);
1553         CRM_ASSERT(rc == pcmk_rc_ok);
1554 
1555         if (print_ip) {
1556             out->message(out, crm_map_element_name(replica->ip->xml), show_opts,
1557                          replica->ip, only_node, only_rsc);
1558         }
1559 
1560         if (print_child) {
1561             out->message(out, crm_map_element_name(replica->child->xml), show_opts,
1562                          replica->child, only_node, only_rsc);
1563         }
1564 
1565         if (print_ctnr) {
1566             out->message(out, crm_map_element_name(replica->container->xml), show_opts,
1567                          replica->container, only_node, only_rsc);
1568         }
1569 
1570         if (print_remote) {
1571             out->message(out, crm_map_element_name(replica->remote->xml), show_opts,
1572                          replica->remote, only_node, only_rsc);
1573         }
1574 
1575         pcmk__output_xml_pop_parent(out); // replica
1576     }
1577 
1578     if (printed_header) {
1579         pcmk__output_xml_pop_parent(out); // bundle
1580     }
1581 
1582     return rc;
1583 }
1584 
1585 static void
1586 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1587                                pe_node_t *node, uint32_t show_opts)
1588 {
1589     pe_resource_t *rsc = replica->child;
1590 
1591     int offset = 0;
1592     char buffer[LINE_MAX];
1593 
1594     if(rsc == NULL) {
1595         rsc = replica->container;
1596     }
1597 
1598     if (replica->remote) {
1599         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1600                            rsc_printable_id(replica->remote));
1601     } else {
1602         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1603                            rsc_printable_id(replica->container));
1604     }
1605     if (replica->ipaddr) {
1606         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1607                            replica->ipaddr);
1608     }
1609 
1610     pe__common_output_html(out, rsc, buffer, node, show_opts);
1611 }
1612 
1613 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1614 int
1615 pe__bundle_html(pcmk__output_t *out, va_list args)
1616 {
1617     uint32_t show_opts = va_arg(args, uint32_t);
1618     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1619     GList *only_node = va_arg(args, GList *);
1620     GList *only_rsc = va_arg(args, GList *);
1621 
1622     pe__bundle_variant_data_t *bundle_data = NULL;
1623     int rc = pcmk_rc_no_output;
1624     gboolean print_everything = TRUE;
1625 
1626     CRM_ASSERT(rsc != NULL);
1627 
1628     get_bundle_variant_data(bundle_data, rsc);
1629 
1630     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1631         return rc;
1632     }
1633 
1634     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1635 
1636     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1637          gIter = gIter->next) {
1638         pe__bundle_replica_t *replica = gIter->data;
1639         gboolean print_ip, print_child, print_ctnr, print_remote;
1640 
1641         CRM_ASSERT(replica);
1642 
1643         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1644             continue;
1645         }
1646 
1647         print_ip = replica->ip != NULL &&
1648                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1649         print_child = replica->child != NULL &&
1650                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1651         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1652         print_remote = replica->remote != NULL &&
1653                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1654 
1655         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1656             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1657             /* The text output messages used below require pe_print_implicit to
1658              * be set to do anything.
1659              */
1660             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1661 
1662             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1663                                      (bundle_data->nreplicas > 1)? " set" : "",
1664                                      rsc->id, bundle_data->image,
1665                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1666                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1667 
1668             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1669                 out->begin_list(out, NULL, NULL, "Replica[%d]", replica->offset);
1670             }
1671 
1672             if (print_ip) {
1673                 out->message(out, crm_map_element_name(replica->ip->xml),
1674                              new_show_opts, replica->ip, only_node, only_rsc);
1675             }
1676 
1677             if (print_child) {
1678                 out->message(out, crm_map_element_name(replica->child->xml),
1679                              new_show_opts, replica->child, only_node, only_rsc);
1680             }
1681 
1682             if (print_ctnr) {
1683                 out->message(out, crm_map_element_name(replica->container->xml),
1684                              new_show_opts, replica->container, only_node, only_rsc);
1685             }
1686 
1687             if (print_remote) {
1688                 out->message(out, crm_map_element_name(replica->remote->xml),
1689                              new_show_opts, replica->remote, only_node, only_rsc);
1690             }
1691 
1692             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1693                 out->end_list(out);
1694             }
1695         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1696             continue;
1697         } else {
1698             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1699                                      (bundle_data->nreplicas > 1)? " set" : "",
1700                                      rsc->id, bundle_data->image,
1701                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1702                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1703 
1704             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1705                                            show_opts);
1706         }
1707     }
1708 
1709     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1710     return rc;
1711 }
1712 
1713 static void
1714 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1715                                pe_node_t *node, uint32_t show_opts)
1716 {
1717     pe_resource_t *rsc = replica->child;
1718 
1719     int offset = 0;
1720     char buffer[LINE_MAX];
1721 
1722     if(rsc == NULL) {
1723         rsc = replica->container;
1724     }
1725 
1726     if (replica->remote) {
1727         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1728                            rsc_printable_id(replica->remote));
1729     } else {
1730         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1731                            rsc_printable_id(replica->container));
1732     }
1733     if (replica->ipaddr) {
1734         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1735                            replica->ipaddr);
1736     }
1737 
1738     pe__common_output_text(out, rsc, buffer, node, show_opts);
1739 }
1740 
1741 PCMK__OUTPUT_ARGS("bundle", "uint32_t", "pe_resource_t *", "GList *", "GList *")
     /*  */
1742 int
1743 pe__bundle_text(pcmk__output_t *out, va_list args)
1744 {
1745     uint32_t show_opts = va_arg(args, uint32_t);
1746     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1747     GList *only_node = va_arg(args, GList *);
1748     GList *only_rsc = va_arg(args, GList *);
1749 
1750     pe__bundle_variant_data_t *bundle_data = NULL;
1751     int rc = pcmk_rc_no_output;
1752     gboolean print_everything = TRUE;
1753 
1754     get_bundle_variant_data(bundle_data, rsc);
1755 
1756     CRM_ASSERT(rsc != NULL);
1757 
1758     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1759         return rc;
1760     }
1761 
1762     print_everything = pcmk__str_in_list(rsc->id, only_rsc, pcmk__str_star_matches);
1763 
1764     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1765          gIter = gIter->next) {
1766         pe__bundle_replica_t *replica = gIter->data;
1767         gboolean print_ip, print_child, print_ctnr, print_remote;
1768 
1769         CRM_ASSERT(replica);
1770 
1771         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1772             continue;
1773         }
1774 
1775         print_ip = replica->ip != NULL &&
1776                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1777         print_child = replica->child != NULL &&
1778                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1779         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1780         print_remote = replica->remote != NULL &&
1781                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1782 
1783         if (pcmk_is_set(show_opts, pcmk_show_implicit_rscs) ||
1784             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1785             /* The text output messages used below require pe_print_implicit to
1786              * be set to do anything.
1787              */
1788             uint32_t new_show_opts = show_opts | pcmk_show_implicit_rscs;
1789 
1790             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1791                                      (bundle_data->nreplicas > 1)? " set" : "",
1792                                      rsc->id, bundle_data->image,
1793                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1794                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1795 
1796             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1797                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1798             }
1799 
1800             out->begin_list(out, NULL, NULL, NULL);
1801 
1802             if (print_ip) {
1803                 out->message(out, crm_map_element_name(replica->ip->xml),
1804                              new_show_opts, replica->ip, only_node, only_rsc);
1805             }
1806 
1807             if (print_child) {
1808                 out->message(out, crm_map_element_name(replica->child->xml),
1809                              new_show_opts, replica->child, only_node, only_rsc);
1810             }
1811 
1812             if (print_ctnr) {
1813                 out->message(out, crm_map_element_name(replica->container->xml),
1814                              new_show_opts, replica->container, only_node, only_rsc);
1815             }
1816 
1817             if (print_remote) {
1818                 out->message(out, crm_map_element_name(replica->remote->xml),
1819                              new_show_opts, replica->remote, only_node, only_rsc);
1820             }
1821 
1822             out->end_list(out);
1823         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1824             continue;
1825         } else {
1826             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1827                                      (bundle_data->nreplicas > 1)? " set" : "",
1828                                      rsc->id, bundle_data->image,
1829                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1830                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1831 
1832             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1833                                            show_opts);
1834         }
1835     }
1836 
1837     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1838     return rc;
1839 }
1840 
1841 static void
1842 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1843                      long options, void *print_data)
1844 {
1845     pe_node_t *node = NULL;
1846     pe_resource_t *rsc = replica->child;
1847 
1848     int offset = 0;
1849     char buffer[LINE_MAX];
1850 
1851     if(rsc == NULL) {
1852         rsc = replica->container;
1853     }
1854 
1855     if (replica->remote) {
1856         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1857                            rsc_printable_id(replica->remote));
1858     } else {
1859         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1860                            rsc_printable_id(replica->container));
1861     }
1862     if (replica->ipaddr) {
1863         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1864                            replica->ipaddr);
1865     }
1866 
1867     node = pe__current_node(replica->container);
1868     common_print(rsc, pre_text, buffer, node, options, print_data);
1869 }
1870 
1871 void
1872 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1873                  void *print_data)
1874 {
1875     pe__bundle_variant_data_t *bundle_data = NULL;
1876     char *child_text = NULL;
1877     CRM_CHECK(rsc != NULL, return);
1878 
1879     if (options & pe_print_xml) {
1880         bundle_print_xml(rsc, pre_text, options, print_data);
1881         return;
1882     }
1883 
1884     get_bundle_variant_data(bundle_data, rsc);
1885 
1886     if (pre_text == NULL) {
1887         pre_text = " ";
1888     }
1889 
1890     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1891                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1892                  rsc->id, bundle_data->image,
1893                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1894                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1895     if (options & pe_print_html) {
1896         status_print("<br />\n<ul>\n");
1897     }
1898 
1899 
1900     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1901          gIter = gIter->next) {
1902         pe__bundle_replica_t *replica = gIter->data;
1903 
1904         CRM_ASSERT(replica);
1905         if (options & pe_print_html) {
1906             status_print("<li>");
1907         }
1908 
1909         if (pcmk_is_set(options, pe_print_implicit)) {
1910             child_text = crm_strdup_printf("     %s", pre_text);
1911             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1912                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1913             }
1914             if (options & pe_print_html) {
1915                 status_print("<br />\n<ul>\n");
1916             }
1917             print_rsc_in_list(replica->ip, child_text, options, print_data);
1918             print_rsc_in_list(replica->container, child_text, options, print_data);
1919             print_rsc_in_list(replica->remote, child_text, options, print_data);
1920             print_rsc_in_list(replica->child, child_text, options, print_data);
1921             if (options & pe_print_html) {
1922                 status_print("</ul>\n");
1923             }
1924         } else {
1925             child_text = crm_strdup_printf("%s  ", pre_text);
1926             print_bundle_replica(replica, child_text, options, print_data);
1927         }
1928         free(child_text);
1929 
1930         if (options & pe_print_html) {
1931             status_print("</li>\n");
1932         }
1933     }
1934     if (options & pe_print_html) {
1935         status_print("</ul>\n");
1936     }
1937 }
1938 
1939 static void
1940 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1941 {
1942     if (replica == NULL) {
1943         return;
1944     }
1945 
1946     if (replica->node) {
1947         free(replica->node);
1948         replica->node = NULL;
1949     }
1950 
1951     if (replica->ip) {
1952         free_xml(replica->ip->xml);
1953         replica->ip->xml = NULL;
1954         replica->ip->fns->free(replica->ip);
1955         replica->ip = NULL;
1956     }
1957     if (replica->container) {
1958         free_xml(replica->container->xml);
1959         replica->container->xml = NULL;
1960         replica->container->fns->free(replica->container);
1961         replica->container = NULL;
1962     }
1963     if (replica->remote) {
1964         free_xml(replica->remote->xml);
1965         replica->remote->xml = NULL;
1966         replica->remote->fns->free(replica->remote);
1967         replica->remote = NULL;
1968     }
1969     free(replica->ipaddr);
1970     free(replica);
1971 }
1972 
1973 void
1974 pe__free_bundle(pe_resource_t *rsc)
     /*  */
1975 {
1976     pe__bundle_variant_data_t *bundle_data = NULL;
1977     CRM_CHECK(rsc != NULL, return);
1978 
1979     get_bundle_variant_data(bundle_data, rsc);
1980     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1981 
1982     free(bundle_data->prefix);
1983     free(bundle_data->image);
1984     free(bundle_data->control_port);
1985     free(bundle_data->host_network);
1986     free(bundle_data->host_netmask);
1987     free(bundle_data->ip_range_start);
1988     free(bundle_data->container_network);
1989     free(bundle_data->launcher_options);
1990     free(bundle_data->container_command);
1991     free(bundle_data->container_host_options);
1992 
1993     g_list_free_full(bundle_data->replicas,
1994                      (GDestroyNotify) free_bundle_replica);
1995     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1996     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1997     g_list_free(rsc->children);
1998 
1999     if(bundle_data->child) {
2000         free_xml(bundle_data->child->xml);
2001         bundle_data->child->xml = NULL;
2002         bundle_data->child->fns->free(bundle_data->child);
2003     }
2004     common_free(rsc);
2005 }
2006 
2007 enum rsc_role_e
2008 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
2009 {
2010     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
2011     return container_role;
2012 }
2013 
2014 /*!
2015  * \brief Get the number of configured replicas in a bundle
2016  *
2017  * \param[in] rsc  Bundle resource
2018  *
2019  * \return Number of configured replicas, or 0 on error
2020  */
2021 int
2022 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
2023 {
2024     if ((rsc == NULL) || (rsc->variant != pe_container)) {
2025         return 0;
2026     } else {
2027         pe__bundle_variant_data_t *bundle_data = NULL;
2028 
2029         get_bundle_variant_data(bundle_data, rsc);
2030         return bundle_data->nreplicas;
2031     }
2032 }
2033 
2034 void
2035 pe__count_bundle(pe_resource_t *rsc)
     /*  */
2036 {
2037     pe__bundle_variant_data_t *bundle_data = NULL;
2038 
2039     get_bundle_variant_data(bundle_data, rsc);
2040     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2041         pe__bundle_replica_t *replica = item->data;
2042 
2043         if (replica->ip) {
2044             replica->ip->fns->count(replica->ip);
2045         }
2046         if (replica->child) {
2047             replica->child->fns->count(replica->child);
2048         }
2049         if (replica->container) {
2050             replica->container->fns->count(replica->container);
2051         }
2052         if (replica->remote) {
2053             replica->remote->fns->count(replica->remote);
2054         }
2055     }
2056 }
2057 
2058 gboolean
2059 pe__bundle_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent)
     /*  */
2060 {
2061     gboolean passes = FALSE;
2062     pe__bundle_variant_data_t *bundle_data = NULL;
2063 
2064     if (pcmk__str_in_list(rsc_printable_id(rsc), only_rsc, pcmk__str_star_matches)) {
2065         passes = TRUE;
2066     } else {
2067         get_bundle_variant_data(bundle_data, rsc);
2068 
2069         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2070             pe__bundle_replica_t *replica = gIter->data;
2071 
2072             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2073                 passes = TRUE;
2074                 break;
2075             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2076                 passes = TRUE;
2077                 break;
2078             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2079                 passes = TRUE;
2080                 break;
2081             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2082                 passes = TRUE;
2083                 break;
2084             }
2085         }
2086     }
2087 
2088     return !passes;
2089 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
