root/lib/pengine/bundle.c

/* */

DEFINITIONS

1. next_ip
2. allocate_ip
3. create_resource
4. valid_network
5. create_ip_resource
6. create_docker_resource
7. create_podman_resource
8. create_rkt_resource
9. disallow_node
10. create_remote_resource
11. create_container
12. mount_add
13. mount_free
14. port_free
15. replica_for_remote
16. pe__bundle_needs_remote_name
17. pe__add_bundle_remote_name
18. pe__unpack_bundle
19. replica_resource_active
20. pe__bundle_active
21. pe__find_bundle_replica
22. print_rsc_in_list
23. container_agent_str
24. bundle_print_xml
25. PCMK__OUTPUT_ARGS
26. pe__bundle_replica_output_html
27. PCMK__OUTPUT_ARGS
28. pe__bundle_replica_output_text
29. PCMK__OUTPUT_ARGS
30. print_bundle_replica
31. pe__print_bundle
32. free_bundle_replica
33. pe__free_bundle
34. pe__bundle_resource_state
35. pe_bundle_replicas
36. pe__count_bundle
37. pe__bundle_is_filtered

   1 /*
   2  * Copyright 2004-2020 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 
  14 #include <crm/pengine/rules.h>
  15 #include <crm/pengine/status.h>
  16 #include <crm/pengine/internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/xml_internal.h>
  19 #include <pe_status_private.h>
  20 
  21 #define PE__VARIANT_BUNDLE 1
  22 #include "./variant.h"
  23 
  24 static char *
  25 next_ip(const char *last_ip)
     /*  */
  26 {
  27     unsigned int oct1 = 0;
  28     unsigned int oct2 = 0;
  29     unsigned int oct3 = 0;
  30     unsigned int oct4 = 0;
  31     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  32 
  33     if (rc != 4) {
  34         /*@ TODO check for IPv6 */
  35         return NULL;
  36 
  37     } else if (oct3 > 253) {
  38         return NULL;
  39 
  40     } else if (oct4 > 253) {
  41         ++oct3;
  42         oct4 = 1;
  43 
  44     } else {
  45         ++oct4;
  46     }
  47 
  48     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  49 }
  50 
  51 static int
  52 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  53             char *buffer, int max)
  54 {
  55     if(data->ip_range_start == NULL) {
  56         return 0;
  57 
  58     } else if(data->ip_last) {
  59         replica->ipaddr = next_ip(data->ip_last);
  60 
  61     } else {
  62         replica->ipaddr = strdup(data->ip_range_start);
  63     }
  64 
  65     data->ip_last = replica->ipaddr;
  66     switch (data->agent_type) {
  67         case PE__CONTAINER_AGENT_DOCKER:
  68         case PE__CONTAINER_AGENT_PODMAN:
  69             if (data->add_host) {
  70                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
  71                                 data->prefix, replica->offset,
  72                                 replica->ipaddr);
  73             }
  74         case PE__CONTAINER_AGENT_RKT:
  75             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
  76                             replica->ipaddr, data->prefix, replica->offset);
  77         default: // PE__CONTAINER_AGENT_UNKNOWN
  78             break;
  79     }
  80     return 0;
  81 }
  82 
  83 static xmlNode *
  84 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
  85 {
  86     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
  87 
  88     crm_xml_add(rsc, XML_ATTR_ID, name);
  89     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
  90     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
  91     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
  92 
  93     return rsc;
  94 }
  95 
  96 /*!
  97  * \internal
  98  * \brief Check whether cluster can manage resource inside container
  99  *
 100  * \param[in] data  Container variant data
 101  *
 102  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 103  *
 104  * \note The resource is manageable if an IP range or control port has been
 105  *       specified. If a control port is used without an IP range, replicas per
 106  *       host must be 1.
 107  */
 108 static bool
 109 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 110 {
 111     if(data->ip_range_start) {
 112         return TRUE;
 113     }
 114     if(data->control_port) {
 115         if(data->nreplicas_per_host > 1) {
 116             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 117             data->nreplicas_per_host = 1;
 118             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 119         }
 120         return TRUE;
 121     }
 122     return FALSE;
 123 }
 124 
 125 static bool
 126 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 127                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 128 {
 129     if(data->ip_range_start) {
 130         char *id = NULL;
 131         xmlNode *xml_ip = NULL;
 132         xmlNode *xml_obj = NULL;
 133 
 134         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 135         crm_xml_sanitize_id(id);
 136         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 137         free(id);
 138 
 139         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 140         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 141                        data->prefix, replica->offset);
 142 
 143         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 144         if(data->host_network) {
 145             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 146         }
 147 
 148         if(data->host_netmask) {
 149             crm_create_nvpair_xml(xml_obj, NULL,
 150                                   "cidr_netmask", data->host_netmask);
 151 
 152         } else {
 153             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 154         }
 155 
 156         xml_obj = create_xml_node(xml_ip, "operations");
 157         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 158 
 159         // TODO: Other ops? Timeouts and intervals from underlying resource?
 160 
 161         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
 162             return FALSE;
 163         }
 164 
 165         parent->children = g_list_append(parent->children, replica->ip);
 166     }
 167     return TRUE;
 168 }
 169 
 170 static bool
 171 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 172                        pe__bundle_replica_t *replica,
 173                        pe_working_set_t *data_set)
 174 {
 175         int offset = 0, max = 4096;
 176         char *buffer = calloc(1, max+1);
 177 
 178         int doffset = 0, dmax = 1024;
 179         char *dbuffer = calloc(1, dmax+1);
 180 
 181         char *id = NULL;
 182         xmlNode *xml_container = NULL;
 183         xmlNode *xml_obj = NULL;
 184 
 185         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
 186         crm_xml_sanitize_id(id);
 187         xml_container = create_resource(id, "heartbeat",
 188                                         PE__CONTAINER_AGENT_DOCKER_S);
 189         free(id);
 190 
 191         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 192         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 193                        data->prefix, replica->offset);
 194 
 195         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 196         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 197         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 198         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 199 
 200         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 201 
 202         /* Set a container hostname only if we have an IP to map it to.
 203          * The user can set -h or --uts=host themselves if they want a nicer
 204          * name for logs, but this makes applications happy who need their
 205          * hostname to match the IP they bind to.
 206          */
 207         if (data->ip_range_start != NULL) {
 208             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 209                                data->prefix, replica->offset);
 210         }
 211 
 212         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 213 
 214         if (data->container_network) {
 215 #if 0
 216             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 217                                replica->ipaddr);
 218 #endif
 219             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 220                                data->container_network);
 221         }
 222 
 223         if(data->control_port) {
 224             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 225         } else {
 226             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 227         }
 228 
 229         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 230             pe__bundle_mount_t *mount = pIter->data;
 231 
 232             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 233                 char *source = crm_strdup_printf(
 234                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 235 
 236                 if(doffset > 0) {
 237                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 238                 }
 239                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 240                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 241                 free(source);
 242 
 243             } else {
 244                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 245             }
 246             if(mount->options) {
 247                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 248             }
 249         }
 250 
 251         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 252             pe__bundle_port_t *port = pIter->data;
 253 
 254             if (replica->ipaddr) {
 255                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 256                                    replica->ipaddr, port->source,
 257                                    port->target);
 258             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 259                 // No need to do port mapping if net=host
 260                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 261             }
 262         }
 263 
 264         if (data->launcher_options) {
 265             offset += snprintf(buffer+offset, max-offset, " %s",
 266                                data->launcher_options);
 267         }
 268 
 269         if (data->container_host_options) {
 270             offset += snprintf(buffer + offset, max - offset, " %s",
 271                                data->container_host_options);
 272         }
 273 
 274         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 275         free(buffer);
 276 
 277         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 278         free(dbuffer);
 279 
 280         if (replica->child) {
 281             if (data->container_command) {
 282                 crm_create_nvpair_xml(xml_obj, NULL,
 283                                       "run_cmd", data->container_command);
 284             } else {
 285                 crm_create_nvpair_xml(xml_obj, NULL,
 286                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 287             }
 288 
 289             /* TODO: Allow users to specify their own?
 290              *
 291              * We just want to know if the container is alive, we'll
 292              * monitor the child independently
 293              */
 294             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 295         /* } else if(child && data->untrusted) {
 296          * Support this use-case?
 297          *
 298          * The ability to have resources started/stopped by us, but
 299          * unable to set attributes, etc.
 300          *
 301          * Arguably better to control API access this with ACLs like
 302          * "normal" remote nodes
 303          *
 304          *     crm_create_nvpair_xml(xml_obj, NULL,
 305          *                           "run_cmd",
 306          *                           "/usr/libexec/pacemaker/pacemaker-execd");
 307          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 308          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
 309          */
 310         } else {
 311             if (data->container_command) {
 312                 crm_create_nvpair_xml(xml_obj, NULL,
 313                                       "run_cmd", data->container_command);
 314             }
 315 
 316             /* TODO: Allow users to specify their own?
 317              *
 318              * We don't know what's in the container, so we just want
 319              * to know if it is alive
 320              */
 321             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 322         }
 323 
 324 
 325         xml_obj = create_xml_node(xml_container, "operations");
 326         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 327 
 328         // TODO: Other ops? Timeouts and intervals from underlying resource?
 329         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 330             return FALSE;
 331         }
 332         parent->children = g_list_append(parent->children, replica->container);
 333         return TRUE;
 334 }
 335 
 336 static bool
 337 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 338                        pe__bundle_replica_t *replica,
 339                        pe_working_set_t *data_set)
 340 {
 341         int offset = 0, max = 4096;
 342         char *buffer = calloc(1, max+1);
 343 
 344         int doffset = 0, dmax = 1024;
 345         char *dbuffer = calloc(1, dmax+1);
 346 
 347         char *id = NULL;
 348         xmlNode *xml_container = NULL;
 349         xmlNode *xml_obj = NULL;
 350 
 351         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
 352         crm_xml_sanitize_id(id);
 353         xml_container = create_resource(id, "heartbeat",
 354                                         PE__CONTAINER_AGENT_PODMAN_S);
 355         free(id);
 356 
 357         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 358         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 359                        data->prefix, replica->offset);
 360 
 361         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 362         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 363         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 364         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 365 
 366         // FIXME: (bandini 2018-08) podman has no restart policies
 367         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 368 
 369         /* Set a container hostname only if we have an IP to map it to.
 370          * The user can set -h or --uts=host themselves if they want a nicer
 371          * name for logs, but this makes applications happy who need their
 372          * hostname to match the IP they bind to.
 373          */
 374         if (data->ip_range_start != NULL) {
 375             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 376                                data->prefix, replica->offset);
 377         }
 378 
 379         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 380 
 381         if (data->container_network) {
 382 #if 0
 383             // podman has no support for --link-local-ip
 384             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 385                                replica->ipaddr);
 386 #endif
 387             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 388                                data->container_network);
 389         }
 390 
 391         if(data->control_port) {
 392             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 393         } else {
 394             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 395         }
 396 
 397         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 398             pe__bundle_mount_t *mount = pIter->data;
 399 
 400             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 401                 char *source = crm_strdup_printf(
 402                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 403 
 404                 if(doffset > 0) {
 405                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 406                 }
 407                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 408                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 409                 free(source);
 410 
 411             } else {
 412                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 413             }
 414             if(mount->options) {
 415                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 416             }
 417         }
 418 
 419         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 420             pe__bundle_port_t *port = pIter->data;
 421 
 422             if (replica->ipaddr) {
 423                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 424                                    replica->ipaddr, port->source,
 425                                    port->target);
 426             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 427                 // No need to do port mapping if net=host
 428                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 429             }
 430         }
 431 
 432         if (data->launcher_options) {
 433             offset += snprintf(buffer+offset, max-offset, " %s",
 434                                data->launcher_options);
 435         }
 436 
 437         if (data->container_host_options) {
 438             offset += snprintf(buffer + offset, max - offset, " %s",
 439                                data->container_host_options);
 440         }
 441 
 442         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 443         free(buffer);
 444 
 445         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 446         free(dbuffer);
 447 
 448         if (replica->child) {
 449             if (data->container_command) {
 450                 crm_create_nvpair_xml(xml_obj, NULL,
 451                                       "run_cmd", data->container_command);
 452             } else {
 453                 crm_create_nvpair_xml(xml_obj, NULL,
 454                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 455             }
 456 
 457             /* TODO: Allow users to specify their own?
 458              *
 459              * We just want to know if the container is alive, we'll
 460              * monitor the child independently
 461              */
 462             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 463         /* } else if(child && data->untrusted) {
 464          * Support this use-case?
 465          *
 466          * The ability to have resources started/stopped by us, but
 467          * unable to set attributes, etc.
 468          *
 469          * Arguably better to control API access this with ACLs like
 470          * "normal" remote nodes
 471          *
 472          *     crm_create_nvpair_xml(xml_obj, NULL,
 473          *                           "run_cmd",
 474          *                           "/usr/libexec/pacemaker/pacemaker-execd");
 475          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 476          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
 477          */
 478         } else {
 479             if (data->container_command) {
 480                 crm_create_nvpair_xml(xml_obj, NULL,
 481                                       "run_cmd", data->container_command);
 482             }
 483 
 484             /* TODO: Allow users to specify their own?
 485              *
 486              * We don't know what's in the container, so we just want
 487              * to know if it is alive
 488              */
 489             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 490         }
 491 
 492 
 493         xml_obj = create_xml_node(xml_container, "operations");
 494         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 495 
 496         // TODO: Other ops? Timeouts and intervals from underlying resource?
 497         if (!common_unpack(xml_container, &replica->container, parent,
 498                            data_set)) {
 499             return FALSE;
 500         }
 501         parent->children = g_list_append(parent->children, replica->container);
 502         return TRUE;
 503 }
 504 
 505 static bool
 506 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 507                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 508 {
 509         int offset = 0, max = 4096;
 510         char *buffer = calloc(1, max+1);
 511 
 512         int doffset = 0, dmax = 1024;
 513         char *dbuffer = calloc(1, dmax+1);
 514 
 515         char *id = NULL;
 516         xmlNode *xml_container = NULL;
 517         xmlNode *xml_obj = NULL;
 518 
 519         int volid = 0;
 520 
 521         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
 522         crm_xml_sanitize_id(id);
 523         xml_container = create_resource(id, "heartbeat",
 524                                         PE__CONTAINER_AGENT_RKT_S);
 525         free(id);
 526 
 527         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 528         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 529                        data->prefix, replica->offset);
 530 
 531         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 532         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
 533         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
 534         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 535 
 536         /* Set a container hostname only if we have an IP to map it to.
 537          * The user can set -h or --uts=host themselves if they want a nicer
 538          * name for logs, but this makes applications happy who need their
 539          * hostname to match the IP they bind to.
 540          */
 541         if (data->ip_range_start != NULL) {
 542             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
 543                                data->prefix, replica->offset);
 544         }
 545 
 546         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 547 
 548         if (data->container_network) {
 549 #if 0
 550             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 551                                replica->ipaddr);
 552 #endif
 553             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 554                                data->container_network);
 555         }
 556 
 557         if(data->control_port) {
 558             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
 559         } else {
 560             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 561         }
 562 
 563         for(GListPtr pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 564             pe__bundle_mount_t *mount = pIter->data;
 565 
 566             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 567                 char *source = crm_strdup_printf(
 568                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 569 
 570                 if(doffset > 0) {
 571                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 572                 }
 573                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 574                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
 575                 if(mount->options) {
 576                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 577                 }
 578                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 579                 free(source);
 580 
 581             } else {
 582                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
 583                 if(mount->options) {
 584                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 585                 }
 586                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 587             }
 588             volid++;
 589         }
 590 
 591         for(GListPtr pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 592             pe__bundle_port_t *port = pIter->data;
 593 
 594             if (replica->ipaddr) {
 595                 offset += snprintf(buffer+offset, max-offset,
 596                                    " --port=%s:%s:%s", port->target,
 597                                    replica->ipaddr, port->source);
 598             } else {
 599                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
 600             }
 601         }
 602 
 603         if (data->launcher_options) {
 604             offset += snprintf(buffer+offset, max-offset, " %s",
 605                                data->launcher_options);
 606         }
 607 
 608         if (data->container_host_options) {
 609             offset += snprintf(buffer + offset, max - offset, " %s",
 610                                data->container_host_options);
 611         }
 612 
 613         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 614         free(buffer);
 615 
 616         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 617         free(dbuffer);
 618 
 619         if (replica->child) {
 620             if (data->container_command) {
 621                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 622                                       data->container_command);
 623             } else {
 624                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 625                                       SBIN_DIR "/pacemaker-remoted");
 626             }
 627 
 628             /* TODO: Allow users to specify their own?
 629              *
 630              * We just want to know if the container is alive, we'll
 631              * monitor the child independently
 632              */
 633             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 634         /* } else if(child && data->untrusted) {
 635          * Support this use-case?
 636          *
 637          * The ability to have resources started/stopped by us, but
 638          * unable to set attributes, etc.
 639          *
 640          * Arguably better to control API access this with ACLs like
 641          * "normal" remote nodes
 642          *
 643          *     crm_create_nvpair_xml(xml_obj, NULL,
 644          *                           "run_cmd",
 645          *                           "/usr/libexec/pacemaker/pacemaker-execd");
 646          *     crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 647          *         "/usr/libexec/pacemaker/lrmd_internal_ctl -c poke");
 648          */
 649         } else {
 650             if (data->container_command) {
 651                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 652                                       data->container_command);
 653             }
 654 
 655             /* TODO: Allow users to specify their own?
 656              *
 657              * We don't know what's in the container, so we just want
 658              * to know if it is alive
 659              */
 660             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 661         }
 662 
 663 
 664         xml_obj = create_xml_node(xml_container, "operations");
 665         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 666 
 667         // TODO: Other ops? Timeouts and intervals from underlying resource?
 668 
 669         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 670             return FALSE;
 671         }
 672         parent->children = g_list_append(parent->children, replica->container);
 673         return TRUE;
 674 }
 675 
 676 /*!
 677  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 678  *
 679  * \param[in,out] rsc    Resource to modify
 680  * \param[in]     uname  Name of node to ban
 681  */
 682 static void
 683 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 684 {
 685     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 686 
 687     if (match) {
 688         ((pe_node_t *) match)->weight = -INFINITY;
 689         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 690     }
 691     if (rsc->children) {
 692         GListPtr child;
 693 
 694         for (child = rsc->children; child != NULL; child = child->next) {
 695             disallow_node((pe_resource_t *) (child->data), uname);
 696         }
 697     }
 698 }
 699 
 700 static bool
 701 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 702                        pe__bundle_replica_t *replica,
 703                        pe_working_set_t *data_set)
 704 {
 705     if (replica->child && valid_network(data)) {
 706         GHashTableIter gIter;
 707         GListPtr rsc_iter = NULL;
 708         pe_node_t *node = NULL;
 709         xmlNode *xml_remote = NULL;
 710         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 711         char *port_s = NULL;
 712         const char *uname = NULL;
 713         const char *connect_name = NULL;
 714 
 715         if (pe_find_resource(data_set->resources, id) != NULL) {
 716             free(id);
 717             // The biggest hammer we have
 718             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 719                                    replica->child->id, replica->offset);
 720             //@TODO return false instead of asserting?
 721             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
 722         }
 723 
 724         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 725          * connection does not have its own IP is a magic string that we use to
 726          * support nested remotes (i.e. a bundle running on a remote node).
 727          */
 728         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 729 
 730         if (data->control_port == NULL) {
 731             port_s = crm_itoa(DEFAULT_REMOTE_PORT);
 732         }
 733 
 734         /* This sets replica->container as replica->remote's container, which is
 735          * similar to what happens with guest nodes. This is how the scheduler
 736          * knows that the bundle node is fenced by recovering the container, and
 737          * that remote should be ordered relative to the container.
 738          */
 739         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 740                                           NULL, NULL, NULL,
 741                                           connect_name, (data->control_port?
 742                                           data->control_port : port_s));
 743         free(port_s);
 744 
 745         /* Abandon our created ID, and pull the copy from the XML, because we
 746          * need something that will get freed during data set cleanup to use as
 747          * the node ID and uname.
 748          */
 749         free(id);
 750         id = NULL;
 751         uname = ID(xml_remote);
 752 
 753         /* Ensure a node has been created for the guest (it may have already
 754          * been, if it has a permanent node attribute), and ensure its weight is
 755          * -INFINITY so no other resources can run on it.
 756          */
 757         node = pe_find_node(data_set->nodes, uname);
 758         if (node == NULL) {
 759             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 760                                   data_set);
 761         } else {
 762             node->weight = -INFINITY;
 763         }
 764         node->rsc_discover_mode = pe_discover_never;
 765 
 766         /* unpack_remote_nodes() ensures that each remote node and guest node
 767          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 768          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 769          * what nodes will be needed, so we do it just above.
 770          *
 771          * Worse, that means that the node may have been utilized while
 772          * unpacking other resources, without our weight correction. The most
 773          * likely place for this to happen is when common_unpack() calls
 774          * resource_location() to set a default score in symmetric clusters.
 775          * This adds a node *copy* to each resource's allowed nodes, and these
 776          * copies will have the wrong weight.
 777          *
 778          * As a hacky workaround, fix those copies here.
 779          *
 780          * @TODO Possible alternative: ensure bundles are unpacked before other
 781          * resources, so the weight is correct before any copies are made.
 782          */
 783         for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
 784             disallow_node((pe_resource_t *) (rsc_iter->data), uname);
 785         }
 786 
 787         replica->node = pe__copy_node(node);
 788         replica->node->weight = 500;
 789         replica->node->rsc_discover_mode = pe_discover_exclusive;
 790 
 791         /* Ensure the node shows up as allowed and with the correct discovery set */
 792         if (replica->child->allowed_nodes != NULL) {
 793             g_hash_table_destroy(replica->child->allowed_nodes);
 794         }
 795         replica->child->allowed_nodes = g_hash_table_new_full(crm_str_hash,
 796                                                               g_str_equal,
 797                                                               NULL, free);
 798         g_hash_table_insert(replica->child->allowed_nodes,
 799                             (gpointer) replica->node->details->id,
 800                             pe__copy_node(replica->node));
 801 
 802         {
 803             pe_node_t *copy = pe__copy_node(replica->node);
 804             copy->weight = -INFINITY;
 805             g_hash_table_insert(replica->child->parent->allowed_nodes,
 806                                 (gpointer) replica->node->details->id, copy);
 807         }
 808         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
 809             return FALSE;
 810         }
 811 
 812         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 813         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 814             if (pe__is_guest_or_remote_node(node)) {
 815                 /* Remote resources can only run on 'normal' cluster node */
 816                 node->weight = -INFINITY;
 817             }
 818         }
 819 
 820         replica->node->details->remote_rsc = replica->remote;
 821 
 822         // Ensure pe__is_guest_node() functions correctly immediately
 823         replica->remote->container = replica->container;
 824 
 825         /* A bundle's #kind is closer to "container" (guest node) than the
 826          * "remote" set by pe_create_node().
 827          */
 828         g_hash_table_insert(replica->node->details->attrs,
 829                             strdup(CRM_ATTR_KIND), strdup("container"));
 830 
 831         /* One effect of this is that setup_container() will add
 832          * replica->remote to replica->container's fillers, which will make
 833          * pe__resource_contains_guest_node() true for replica->container.
 834          *
 835          * replica->child does NOT get added to replica->container's fillers.
 836          * The only noticeable effect if it did would be for its fail count to
 837          * be taken into account when checking replica->container's migration
 838          * threshold.
 839          */
 840         parent->children = g_list_append(parent->children, replica->remote);
 841     }
 842     return TRUE;
 843 }
 844 
 845 static bool
 846 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 847                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 848 {
 849 
 850     switch (data->agent_type) {
 851         case PE__CONTAINER_AGENT_DOCKER:
 852             if (!create_docker_resource(parent, data, replica, data_set)) {
 853                 return FALSE;
 854             }
 855             break;
 856 
 857         case PE__CONTAINER_AGENT_PODMAN:
 858             if (!create_podman_resource(parent, data, replica, data_set)) {
 859                 return FALSE;
 860             }
 861             break;
 862 
 863         case PE__CONTAINER_AGENT_RKT:
 864             if (!create_rkt_resource(parent, data, replica, data_set)) {
 865                 return FALSE;
 866             }
 867             break;
 868         default: // PE__CONTAINER_AGENT_UNKNOWN
 869             return FALSE;
 870     }
 871 
 872     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
 873         return FALSE;
 874     }
 875     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
 876         return FALSE;
 877     }
 878     if (replica->child && replica->ipaddr) {
 879         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 880     }
 881 
 882     if (replica->remote) {
 883         /*
 884          * Allow the remote connection resource to be allocated to a
 885          * different node than the one on which the container is active.
 886          *
 887          * This makes it possible to have Pacemaker Remote nodes running
 888          * containers with pacemaker-remoted inside in order to start
 889          * services inside those containers.
 890          */
 891         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 892     }
 893 
 894     return TRUE;
 895 }
 896 
 897 static void
 898 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 899           const char *target, const char *options, uint32_t flags)
 900 {
 901     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 902 
 903     mount->source = strdup(source);
 904     mount->target = strdup(target);
 905     if (options) {
 906         mount->options = strdup(options);
 907     }
 908     mount->flags = flags;
 909     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 910 }
 911 
 912 static void
 913 mount_free(pe__bundle_mount_t *mount)
     /*  */
 914 {
 915     free(mount->source);
 916     free(mount->target);
 917     free(mount->options);
 918     free(mount);
 919 }
 920 
 921 static void
 922 port_free(pe__bundle_port_t *port)
     /*  */
 923 {
 924     free(port->source);
 925     free(port->target);
 926     free(port);
 927 }
 928 
 929 static pe__bundle_replica_t *
 930 replica_for_remote(pe_resource_t *remote)
     /*  */
 931 {
 932     pe_resource_t *top = remote;
 933     pe__bundle_variant_data_t *bundle_data = NULL;
 934 
 935     if (top == NULL) {
 936         return NULL;
 937     }
 938 
 939     while (top->parent != NULL) {
 940         top = top->parent;
 941     }
 942 
 943     get_bundle_variant_data(bundle_data, top);
 944     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 945          gIter = gIter->next) {
 946         pe__bundle_replica_t *replica = gIter->data;
 947 
 948         if (replica->remote == remote) {
 949             return replica;
 950         }
 951     }
 952     CRM_LOG_ASSERT(FALSE);
 953     return NULL;
 954 }
 955 
 956 bool
 957 pe__bundle_needs_remote_name(pe_resource_t *rsc)
     /*  */
 958 {
 959     const char *value;
 960 
 961     if (rsc == NULL) {
 962         return FALSE;
 963     }
 964 
 965     value = g_hash_table_lookup(rsc->parameters, XML_RSC_ATTR_REMOTE_RA_ADDR);
 966     if (!pcmk__str_eq(value, "#uname", pcmk__str_casei)) {
 967         return FALSE;
 968 
 969     } else {
 970         const char *match[3][2] = {
 971             { XML_ATTR_TYPE,           "remote"                },
 972             { XML_AGENT_ATTR_CLASS,    PCMK_RESOURCE_CLASS_OCF },
 973             { XML_AGENT_ATTR_PROVIDER, "pacemaker"             },
 974         };
 975 
 976         for (int m = 0; m < 3; m++) {
 977             value = crm_element_value(rsc->xml, match[m][0]);
 978             if (!pcmk__str_eq(value, match[m][1], pcmk__str_casei)) {
 979                 return FALSE;
 980             }
 981         }
 982     }
 983     return TRUE;
 984 }
 985 
 986 const char *
 987 pe__add_bundle_remote_name(pe_resource_t *rsc, xmlNode *xml, const char *field)
     /*  */
 988 {
 989     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 990 
 991     pe_node_t *node = NULL;
 992     pe__bundle_replica_t *replica = NULL;
 993 
 994     if (!pe__bundle_needs_remote_name(rsc)) {
 995         return NULL;
 996     }
 997 
 998     replica = replica_for_remote(rsc);
 999     if (replica == NULL) {
1000         return NULL;
1001     }
1002 
1003     node = replica->container->allocated_to;
1004     if (node == NULL) {
1005         /* If it won't be running anywhere after the
1006          * transition, go with where it's running now.
1007          */
1008         node = pe__current_node(replica->container);
1009     }
1010 
1011     if(node == NULL) {
1012         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
1013         return NULL;
1014     }
1015 
1016     crm_trace("Setting address for bundle connection %s to bundle host %s",
1017               rsc->id, node->details->uname);
1018     if(xml != NULL && field != NULL) {
1019         crm_xml_add(xml, field, node->details->uname);
1020     }
1021 
1022     return node->details->uname;
1023 }
1024 
1025 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
1026         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
1027                                    "Bundle mount", ID(mount_xml), flags,    \
1028                                    (flags_to_set), #flags_to_set);          \
1029     } while (0)
1030 
1031 gboolean
1032 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
1033 {
1034     const char *value = NULL;
1035     xmlNode *xml_obj = NULL;
1036     xmlNode *xml_resource = NULL;
1037     pe__bundle_variant_data_t *bundle_data = NULL;
1038     bool need_log_mount = TRUE;
1039 
1040     CRM_ASSERT(rsc != NULL);
1041     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
1042 
1043     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
1044     rsc->variant_opaque = bundle_data;
1045     bundle_data->prefix = strdup(rsc->id);
1046 
1047     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1048     if (xml_obj != NULL) {
1049         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1050     } else {
1051         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1052         if (xml_obj != NULL) {
1053             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1054         } else {
1055             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1056             if (xml_obj != NULL) {
1057                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1058             } else {
1059                 return FALSE;
1060             }
1061         }
1062     }
1063 
1064     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
1065     if (value == NULL) {
1066         // @COMPAT deprecated since 2.0.0
1067         value = crm_element_value(xml_obj, "masters");
1068     }
1069     bundle_data->promoted_max = crm_parse_int(value, "0");
1070     if (bundle_data->promoted_max < 0) {
1071         pe_err("%s for %s must be nonnegative integer, using 0",
1072                XML_RSC_ATTR_PROMOTED_MAX, rsc->id);
1073         bundle_data->promoted_max = 0;
1074     }
1075 
1076     value = crm_element_value(xml_obj, "replicas");
1077     if ((value == NULL) && bundle_data->promoted_max) {
1078         bundle_data->nreplicas = bundle_data->promoted_max;
1079     } else {
1080         bundle_data->nreplicas = crm_parse_int(value, "1");
1081     }
1082     if (bundle_data->nreplicas < 1) {
1083         pe_err("'replicas' for %s must be positive integer, using 1", rsc->id);
1084         bundle_data->nreplicas = 1;
1085     }
1086 
1087     /*
1088      * Communication between containers on the same host via the
1089      * floating IPs only works if the container is started with:
1090      *   --userland-proxy=false --ip-masq=false
1091      */
1092     value = crm_element_value(xml_obj, "replicas-per-host");
1093     bundle_data->nreplicas_per_host = crm_parse_int(value, "1");
1094     if (bundle_data->nreplicas_per_host < 1) {
1095         pe_err("'replicas-per-host' for %s must be positive integer, using 1",
1096                rsc->id);
1097         bundle_data->nreplicas_per_host = 1;
1098     }
1099     if (bundle_data->nreplicas_per_host == 1) {
1100         pe__clear_resource_flags(rsc, pe_rsc_unique);
1101     }
1102 
1103     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1104     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1105     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1106     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1107 
1108     xml_obj = first_named_child(rsc->xml, "network");
1109     if(xml_obj) {
1110 
1111         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1112         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1113         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1114         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1115         value = crm_element_value(xml_obj, "add-host");
1116         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1117             bundle_data->add_host = TRUE;
1118         }
1119 
1120         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1121              xml_child = pcmk__xe_next(xml_child)) {
1122 
1123             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1124             port->source = crm_element_value_copy(xml_child, "port");
1125 
1126             if(port->source == NULL) {
1127                 port->source = crm_element_value_copy(xml_child, "range");
1128             } else {
1129                 port->target = crm_element_value_copy(xml_child, "internal-port");
1130             }
1131 
1132             if(port->source != NULL && strlen(port->source) > 0) {
1133                 if(port->target == NULL) {
1134                     port->target = strdup(port->source);
1135                 }
1136                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1137 
1138             } else {
1139                 pe_err("Invalid port directive %s", ID(xml_child));
1140                 port_free(port);
1141             }
1142         }
1143     }
1144 
1145     xml_obj = first_named_child(rsc->xml, "storage");
1146     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1147          xml_child = pcmk__xe_next(xml_child)) {
1148 
1149         const char *source = crm_element_value(xml_child, "source-dir");
1150         const char *target = crm_element_value(xml_child, "target-dir");
1151         const char *options = crm_element_value(xml_child, "options");
1152         int flags = pe__bundle_mount_none;
1153 
1154         if (source == NULL) {
1155             source = crm_element_value(xml_child, "source-dir-root");
1156             pe__set_bundle_mount_flags(xml_child, flags,
1157                                        pe__bundle_mount_subdir);
1158         }
1159 
1160         if (source && target) {
1161             mount_add(bundle_data, source, target, options, flags);
1162             if (strcmp(target, "/var/log") == 0) {
1163                 need_log_mount = FALSE;
1164             }
1165         } else {
1166             pe_err("Invalid mount directive %s", ID(xml_child));
1167         }
1168     }
1169 
1170     xml_obj = first_named_child(rsc->xml, "primitive");
1171     if (xml_obj && valid_network(bundle_data)) {
1172         char *value = NULL;
1173         xmlNode *xml_set = NULL;
1174 
1175         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1176 
1177         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1178          * part of the resource name, so that bundles don't restart in a rolling
1179          * upgrade. (It also avoids needing to change regression tests.)
1180          */
1181         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1182                       (bundle_data->promoted_max? "master"
1183                       : (const char *)xml_resource->name));
1184 
1185         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1186         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1187 
1188         crm_create_nvpair_xml(xml_set, NULL,
1189                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1190 
1191         value = crm_itoa(bundle_data->nreplicas);
1192         crm_create_nvpair_xml(xml_set, NULL,
1193                               XML_RSC_ATTR_INCARNATION_MAX, value);
1194         free(value);
1195 
1196         value = crm_itoa(bundle_data->nreplicas_per_host);
1197         crm_create_nvpair_xml(xml_set, NULL,
1198                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
1199         free(value);
1200 
1201         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1202                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1203 
1204         if (bundle_data->promoted_max) {
1205             crm_create_nvpair_xml(xml_set, NULL,
1206                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1207 
1208             value = crm_itoa(bundle_data->promoted_max);
1209             crm_create_nvpair_xml(xml_set, NULL,
1210                                   XML_RSC_ATTR_PROMOTED_MAX, value);
1211             free(value);
1212         }
1213 
1214         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1215         add_node_copy(xml_resource, xml_obj);
1216 
1217     } else if(xml_obj) {
1218         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1219                rsc->id, ID(xml_obj));
1220         return FALSE;
1221     }
1222 
1223     if(xml_resource) {
1224         int lpc = 0;
1225         GListPtr childIter = NULL;
1226         pe_resource_t *new_rsc = NULL;
1227         pe__bundle_port_t *port = NULL;
1228 
1229         int offset = 0, max = 1024;
1230         char *buffer = NULL;
1231 
1232         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
1233             pe_err("Failed unpacking resource %s", ID(rsc->xml));
1234             if (new_rsc != NULL && new_rsc->fns != NULL) {
1235                 new_rsc->fns->free(new_rsc);
1236             }
1237             return FALSE;
1238         }
1239 
1240         bundle_data->child = new_rsc;
1241 
1242         /* Currently, we always map the default authentication key location
1243          * into the same location inside the container.
1244          *
1245          * Ideally, we would respect the host's PCMK_authkey_location, but:
1246          * - it may be different on different nodes;
1247          * - the actual connection will do extra checking to make sure the key
1248          *   file exists and is readable, that we can't do here on the DC
1249          * - tools such as crm_resource and crm_simulate may not have the same
1250          *   environment variables as the cluster, causing operation digests to
1251          *   differ
1252          *
1253          * Always using the default location inside the container is fine,
1254          * because we control the pacemaker_remote environment, and it avoids
1255          * having to pass another environment variable to the container.
1256          *
1257          * @TODO A better solution may be to have only pacemaker_remote use the
1258          * environment variable, and have the cluster nodes use a new
1259          * cluster option for key location. This would introduce the limitation
1260          * of the location being the same on all cluster nodes, but that's
1261          * reasonable.
1262          */
1263         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1264                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1265 
1266         if (need_log_mount) {
1267             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1268                       pe__bundle_mount_subdir);
1269         }
1270 
1271         port = calloc(1, sizeof(pe__bundle_port_t));
1272         if(bundle_data->control_port) {
1273             port->source = strdup(bundle_data->control_port);
1274         } else {
1275             /* If we wanted to respect PCMK_remote_port, we could use
1276              * crm_default_remote_port() here and elsewhere in this file instead
1277              * of DEFAULT_REMOTE_PORT.
1278              *
1279              * However, it gains nothing, since we control both the container
1280              * environment and the connection resource parameters, and the user
1281              * can use a different port if desired by setting control-port.
1282              */
1283             port->source = crm_itoa(DEFAULT_REMOTE_PORT);
1284         }
1285         port->target = strdup(port->source);
1286         bundle_data->ports = g_list_append(bundle_data->ports, port);
1287 
1288         buffer = calloc(1, max+1);
1289         for (childIter = bundle_data->child->children; childIter != NULL;
1290              childIter = childIter->next) {
1291 
1292             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1293 
1294             replica->child = childIter->data;
1295             replica->child->exclusive_discover = TRUE;
1296             replica->offset = lpc++;
1297 
1298             // Ensure the child's notify gets set based on the underlying primitive's value
1299             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1300                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1301             }
1302 
1303             offset += allocate_ip(bundle_data, replica, buffer+offset,
1304                                   max-offset);
1305             bundle_data->replicas = g_list_append(bundle_data->replicas,
1306                                                   replica);
1307             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1308                                                                 XML_RSC_ATTR_TARGET);
1309         }
1310         bundle_data->container_host_options = buffer;
1311         if (bundle_data->attribute_target) {
1312             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1313                                  strdup(bundle_data->attribute_target));
1314             g_hash_table_replace(bundle_data->child->meta,
1315                                  strdup(XML_RSC_ATTR_TARGET),
1316                                  strdup(bundle_data->attribute_target));
1317         }
1318 
1319     } else {
1320         // Just a naked container, no pacemaker-remote
1321         int offset = 0, max = 1024;
1322         char *buffer = calloc(1, max+1);
1323 
1324         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1325             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1326 
1327             replica->offset = lpc;
1328             offset += allocate_ip(bundle_data, replica, buffer+offset,
1329                                   max-offset);
1330             bundle_data->replicas = g_list_append(bundle_data->replicas,
1331                                                   replica);
1332         }
1333         bundle_data->container_host_options = buffer;
1334     }
1335 
1336     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1337          gIter = gIter->next) {
1338         pe__bundle_replica_t *replica = gIter->data;
1339 
1340         if (!create_container(rsc, bundle_data, replica, data_set)) {
1341             pe_err("Failed unpacking resource %s", rsc->id);
1342             rsc->fns->free(rsc);
1343             return FALSE;
1344         }
1345     }
1346 
1347     if (bundle_data->child) {
1348         rsc->children = g_list_append(rsc->children, bundle_data->child);
1349     }
1350     return TRUE;
1351 }
1352 
1353 static int
1354 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1355 {
1356     if (rsc) {
1357         gboolean child_active = rsc->fns->active(rsc, all);
1358 
1359         if (child_active && !all) {
1360             return TRUE;
1361         } else if (!child_active && all) {
1362             return FALSE;
1363         }
1364     }
1365     return -1;
1366 }
1367 
1368 gboolean
1369 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1370 {
1371     pe__bundle_variant_data_t *bundle_data = NULL;
1372     GListPtr iter = NULL;
1373 
1374     get_bundle_variant_data(bundle_data, rsc);
1375     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1376         pe__bundle_replica_t *replica = iter->data;
1377         int rsc_active;
1378 
1379         rsc_active = replica_resource_active(replica->ip, all);
1380         if (rsc_active >= 0) {
1381             return (gboolean) rsc_active;
1382         }
1383 
1384         rsc_active = replica_resource_active(replica->child, all);
1385         if (rsc_active >= 0) {
1386             return (gboolean) rsc_active;
1387         }
1388 
1389         rsc_active = replica_resource_active(replica->container, all);
1390         if (rsc_active >= 0) {
1391             return (gboolean) rsc_active;
1392         }
1393 
1394         rsc_active = replica_resource_active(replica->remote, all);
1395         if (rsc_active >= 0) {
1396             return (gboolean) rsc_active;
1397         }
1398     }
1399 
1400     /* If "all" is TRUE, we've already checked that no resources were inactive,
1401      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1402      * so return FALSE.
1403      */
1404     return all;
1405 }
1406 
1407 /*!
1408  * \internal
1409  * \brief Find the bundle replica corresponding to a given node
1410  *
1411  * \param[in] bundle  Top-level bundle resource
1412  * \param[in] node    Node to search for
1413  *
1414  * \return Bundle replica if found, NULL otherwise
1415  */
1416 pe_resource_t *
1417 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1418 {
1419     pe__bundle_variant_data_t *bundle_data = NULL;
1420     CRM_ASSERT(bundle && node);
1421 
1422     get_bundle_variant_data(bundle_data, bundle);
1423     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1424          gIter = gIter->next) {
1425         pe__bundle_replica_t *replica = gIter->data;
1426 
1427         CRM_ASSERT(replica && replica->node);
1428         if (replica->node->details == node->details) {
1429             return replica->child;
1430         }
1431     }
1432     return NULL;
1433 }
1434 
1435 static void
1436 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1437                   void *print_data)
1438 {
1439     if (rsc != NULL) {
1440         if (options & pe_print_html) {
1441             status_print("<li>");
1442         }
1443         rsc->fns->print(rsc, pre_text, options, print_data);
1444         if (options & pe_print_html) {
1445             status_print("</li>\n");
1446         }
1447     }
1448 }
1449 
1450 static const char*
1451 container_agent_str(enum pe__container_agent t)
     /*  */
1452 {
1453     switch (t) {
1454         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
1455         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
1456         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
1457         default: // PE__CONTAINER_AGENT_UNKNOWN
1458             break;
1459     }
1460     return PE__CONTAINER_AGENT_UNKNOWN_S;
1461 }
1462 
1463 static void
1464 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1465                  void *print_data)
1466 {
1467     pe__bundle_variant_data_t *bundle_data = NULL;
1468     char *child_text = NULL;
1469     CRM_CHECK(rsc != NULL, return);
1470 
1471     if (pre_text == NULL) {
1472         pre_text = "";
1473     }
1474     child_text = crm_strdup_printf("%s        ", pre_text);
1475 
1476     get_bundle_variant_data(bundle_data, rsc);
1477 
1478     status_print("%s<bundle ", pre_text);
1479     status_print("id=\"%s\" ", rsc->id);
1480     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1481     status_print("image=\"%s\" ", bundle_data->image);
1482     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1483     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1484     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1485     status_print(">\n");
1486 
1487     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1488          gIter = gIter->next) {
1489         pe__bundle_replica_t *replica = gIter->data;
1490 
1491         CRM_ASSERT(replica);
1492         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
1493         print_rsc_in_list(replica->ip, child_text, options, print_data);
1494         print_rsc_in_list(replica->child, child_text, options, print_data);
1495         print_rsc_in_list(replica->container, child_text, options, print_data);
1496         print_rsc_in_list(replica->remote, child_text, options, print_data);
1497         status_print("%s    </replica>\n", pre_text);
1498     }
1499     status_print("%s</bundle>\n", pre_text);
1500     free(child_text);
1501 }
1502 
1503 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GListPtr", "GListPtr")
     /*  */
1504 int
1505 pe__bundle_xml(pcmk__output_t *out, va_list args)
1506 {
1507     unsigned int options = va_arg(args, unsigned int);
1508     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1509     GListPtr only_node = va_arg(args, GListPtr);
1510     GListPtr only_rsc = va_arg(args, GListPtr);
1511 
1512     pe__bundle_variant_data_t *bundle_data = NULL;
1513     int rc = pcmk_rc_no_output;
1514     gboolean printed_header = FALSE;
1515     gboolean print_everything = TRUE;
1516 
1517     CRM_ASSERT(rsc != NULL);
1518 
1519     get_bundle_variant_data(bundle_data, rsc);
1520 
1521     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1522         return rc;
1523     }
1524 
1525     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1526 
1527     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1528          gIter = gIter->next) {
1529         pe__bundle_replica_t *replica = gIter->data;
1530         char *id = crm_itoa(replica->offset);
1531         gboolean print_ip, print_child, print_ctnr, print_remote;
1532 
1533         CRM_ASSERT(replica);
1534 
1535         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1536             continue;
1537         }
1538 
1539         print_ip = replica->ip != NULL &&
1540                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1541         print_child = replica->child != NULL &&
1542                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1543         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1544         print_remote = replica->remote != NULL &&
1545                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1546 
1547         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1548             continue;
1549         }
1550 
1551         if (!printed_header) {
1552             printed_header = TRUE;
1553 
1554             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6,
1555                      "id", rsc->id,
1556                      "type", container_agent_str(bundle_data->agent_type),
1557                      "image", bundle_data->image,
1558                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1559                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1560                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed));
1561             CRM_ASSERT(rc == pcmk_rc_ok);
1562         }
1563 
1564         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1565         free(id);
1566         CRM_ASSERT(rc == pcmk_rc_ok);
1567 
1568         if (print_ip) {
1569             out->message(out, crm_map_element_name(replica->ip->xml), options,
1570                          replica->ip, only_node, only_rsc);
1571         }
1572 
1573         if (print_child) {
1574             out->message(out, crm_map_element_name(replica->child->xml), options,
1575                          replica->child, only_node, only_rsc);
1576         }
1577 
1578         if (print_ctnr) {
1579             out->message(out, crm_map_element_name(replica->container->xml), options,
1580                          replica->container, only_node, only_rsc);
1581         }
1582 
1583         if (print_remote) {
1584             out->message(out, crm_map_element_name(replica->remote->xml), options,
1585                          replica->remote, only_node, only_rsc);
1586         }
1587 
1588         pcmk__output_xml_pop_parent(out); // replica
1589     }
1590 
1591     if (printed_header) {
1592         pcmk__output_xml_pop_parent(out); // bundle
1593     }
1594 
1595     return rc;
1596 }
1597 
1598 static void
1599 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1600                                pe_node_t *node, long options)
1601 {
1602     pe_resource_t *rsc = replica->child;
1603 
1604     int offset = 0;
1605     char buffer[LINE_MAX];
1606 
1607     if(rsc == NULL) {
1608         rsc = replica->container;
1609     }
1610 
1611     if (replica->remote) {
1612         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1613                            rsc_printable_id(replica->remote));
1614     } else {
1615         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1616                            rsc_printable_id(replica->container));
1617     }
1618     if (replica->ipaddr) {
1619         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1620                            replica->ipaddr);
1621     }
1622 
1623     pe__common_output_html(out, rsc, buffer, node, options);
1624 }
1625 
1626 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GListPtr", "GListPtr")
     /*  */
1627 int
1628 pe__bundle_html(pcmk__output_t *out, va_list args)
1629 {
1630     unsigned int options = va_arg(args, unsigned int);
1631     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1632     GListPtr only_node = va_arg(args, GListPtr);
1633     GListPtr only_rsc = va_arg(args, GListPtr);
1634 
1635     pe__bundle_variant_data_t *bundle_data = NULL;
1636     char buffer[LINE_MAX];
1637     int rc = pcmk_rc_no_output;
1638     gboolean print_everything = TRUE;
1639 
1640     CRM_ASSERT(rsc != NULL);
1641 
1642     get_bundle_variant_data(bundle_data, rsc);
1643 
1644     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1645         return rc;
1646     }
1647 
1648     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1649 
1650     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1651          gIter = gIter->next) {
1652         pe__bundle_replica_t *replica = gIter->data;
1653         gboolean print_ip, print_child, print_ctnr, print_remote;
1654 
1655         CRM_ASSERT(replica);
1656 
1657         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1658             continue;
1659         }
1660 
1661         print_ip = replica->ip != NULL &&
1662                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1663         print_child = replica->child != NULL &&
1664                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1665         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1666         print_remote = replica->remote != NULL &&
1667                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1668 
1669         if (pcmk_is_set(options, pe_print_implicit) ||
1670             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1671             /* The text output messages used below require pe_print_implicit to
1672              * be set to do anything.
1673              */
1674             unsigned int new_options = options;
1675 
1676             if (!pcmk_is_set(options, pe_print_implicit)) {
1677                 new_options |= pe_print_implicit;
1678             }
1679 
1680             if (rc == pcmk_rc_no_output) {
1681                 pcmk__output_create_xml_node(out, "br");
1682             }
1683 
1684             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1685                                      (bundle_data->nreplicas > 1)? " set" : "",
1686                                      rsc->id, bundle_data->image,
1687                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1688                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1689 
1690             pcmk__output_xml_create_parent(out, "li");
1691 
1692             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1693                 snprintf(buffer, LINE_MAX, " Replica[%d]", replica->offset);
1694                 xmlNodeSetContent(pcmk__output_xml_peek_parent(out), (pcmkXmlStr) buffer);
1695             }
1696             pcmk__output_create_xml_node(out, "br");
1697             out->begin_list(out, NULL, NULL, NULL);
1698 
1699             if (print_ip) {
1700                 out->message(out, crm_map_element_name(replica->ip->xml),
1701                              new_options, replica->ip, only_node, only_rsc);
1702             }
1703 
1704             if (print_child) {
1705                 out->message(out, crm_map_element_name(replica->child->xml),
1706                              new_options, replica->child, only_node, only_rsc);
1707             }
1708 
1709             if (print_ctnr) {
1710                 out->message(out, crm_map_element_name(replica->container->xml),
1711                              new_options, replica->container, only_node, only_rsc);
1712             }
1713 
1714             if (print_remote) {
1715                 out->message(out, crm_map_element_name(replica->remote->xml),
1716                              new_options, replica->remote, only_node, only_rsc);
1717             }
1718 
1719             out->end_list(out);
1720         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1721             continue;
1722         } else {
1723             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1724                                      (bundle_data->nreplicas > 1)? " set" : "",
1725                                      rsc->id, bundle_data->image,
1726                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1727                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1728 
1729             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1730                                            options);
1731         }
1732 
1733         pcmk__output_xml_pop_parent(out);
1734     }
1735 
1736     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1737     return rc;
1738 }
1739 
1740 static void
1741 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1742                                pe_node_t *node, long options)
1743 {
1744     pe_resource_t *rsc = replica->child;
1745 
1746     int offset = 0;
1747     char buffer[LINE_MAX];
1748 
1749     if(rsc == NULL) {
1750         rsc = replica->container;
1751     }
1752 
1753     if (replica->remote) {
1754         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1755                            rsc_printable_id(replica->remote));
1756     } else {
1757         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1758                            rsc_printable_id(replica->container));
1759     }
1760     if (replica->ipaddr) {
1761         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1762                            replica->ipaddr);
1763     }
1764 
1765     pe__common_output_text(out, rsc, buffer, node, options);
1766 }
1767 
1768 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GListPtr", "GListPtr")
     /*  */
1769 int
1770 pe__bundle_text(pcmk__output_t *out, va_list args)
1771 {
1772     unsigned int options = va_arg(args, unsigned int);
1773     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1774     GListPtr only_node = va_arg(args, GListPtr);
1775     GListPtr only_rsc = va_arg(args, GListPtr);
1776 
1777     pe__bundle_variant_data_t *bundle_data = NULL;
1778     int rc = pcmk_rc_no_output;
1779     gboolean print_everything = TRUE;
1780 
1781     get_bundle_variant_data(bundle_data, rsc);
1782 
1783     CRM_ASSERT(rsc != NULL);
1784 
1785     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1786         return rc;
1787     }
1788 
1789     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1790 
1791     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1792          gIter = gIter->next) {
1793         pe__bundle_replica_t *replica = gIter->data;
1794         gboolean print_ip, print_child, print_ctnr, print_remote;
1795 
1796         CRM_ASSERT(replica);
1797 
1798         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1799             continue;
1800         }
1801 
1802         print_ip = replica->ip != NULL &&
1803                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1804         print_child = replica->child != NULL &&
1805                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1806         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1807         print_remote = replica->remote != NULL &&
1808                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1809 
1810         if (pcmk_is_set(options, pe_print_implicit) ||
1811             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1812             /* The text output messages used below require pe_print_implicit to
1813              * be set to do anything.
1814              */
1815             unsigned int new_options = options;
1816 
1817             if (!pcmk_is_set(options, pe_print_implicit)) {
1818                 new_options |= pe_print_implicit;
1819             }
1820             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1821                                      (bundle_data->nreplicas > 1)? " set" : "",
1822                                      rsc->id, bundle_data->image,
1823                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1824                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1825 
1826             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1827                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1828             }
1829 
1830             out->begin_list(out, NULL, NULL, NULL);
1831 
1832             if (print_ip) {
1833                 out->message(out, crm_map_element_name(replica->ip->xml),
1834                              new_options, replica->ip, only_node, only_rsc);
1835             }
1836 
1837             if (print_child) {
1838                 out->message(out, crm_map_element_name(replica->child->xml),
1839                              new_options, replica->child, only_node, only_rsc);
1840             }
1841 
1842             if (print_ctnr) {
1843                 out->message(out, crm_map_element_name(replica->container->xml),
1844                              new_options, replica->container, only_node, only_rsc);
1845             }
1846 
1847             if (print_remote) {
1848                 out->message(out, crm_map_element_name(replica->remote->xml),
1849                              new_options, replica->remote, only_node, only_rsc);
1850             }
1851 
1852             out->end_list(out);
1853         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1854             continue;
1855         } else {
1856             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1857                                      (bundle_data->nreplicas > 1)? " set" : "",
1858                                      rsc->id, bundle_data->image,
1859                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1860                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1861 
1862             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1863                                            options);
1864         }
1865     }
1866 
1867     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1868     return rc;
1869 }
1870 
1871 static void
1872 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1873                      long options, void *print_data)
1874 {
1875     pe_node_t *node = NULL;
1876     pe_resource_t *rsc = replica->child;
1877 
1878     int offset = 0;
1879     char buffer[LINE_MAX];
1880 
1881     if(rsc == NULL) {
1882         rsc = replica->container;
1883     }
1884 
1885     if (replica->remote) {
1886         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1887                            rsc_printable_id(replica->remote));
1888     } else {
1889         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1890                            rsc_printable_id(replica->container));
1891     }
1892     if (replica->ipaddr) {
1893         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1894                            replica->ipaddr);
1895     }
1896 
1897     node = pe__current_node(replica->container);
1898     common_print(rsc, pre_text, buffer, node, options, print_data);
1899 }
1900 
1901 void
1902 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1903                  void *print_data)
1904 {
1905     pe__bundle_variant_data_t *bundle_data = NULL;
1906     char *child_text = NULL;
1907     CRM_CHECK(rsc != NULL, return);
1908 
1909     if (options & pe_print_xml) {
1910         bundle_print_xml(rsc, pre_text, options, print_data);
1911         return;
1912     }
1913 
1914     get_bundle_variant_data(bundle_data, rsc);
1915 
1916     if (pre_text == NULL) {
1917         pre_text = " ";
1918     }
1919 
1920     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1921                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1922                  rsc->id, bundle_data->image,
1923                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1924                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1925     if (options & pe_print_html) {
1926         status_print("<br />\n<ul>\n");
1927     }
1928 
1929 
1930     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1931          gIter = gIter->next) {
1932         pe__bundle_replica_t *replica = gIter->data;
1933 
1934         CRM_ASSERT(replica);
1935         if (options & pe_print_html) {
1936             status_print("<li>");
1937         }
1938 
1939         if (pcmk_is_set(options, pe_print_implicit)) {
1940             child_text = crm_strdup_printf("     %s", pre_text);
1941             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1942                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1943             }
1944             if (options & pe_print_html) {
1945                 status_print("<br />\n<ul>\n");
1946             }
1947             print_rsc_in_list(replica->ip, child_text, options, print_data);
1948             print_rsc_in_list(replica->container, child_text, options, print_data);
1949             print_rsc_in_list(replica->remote, child_text, options, print_data);
1950             print_rsc_in_list(replica->child, child_text, options, print_data);
1951             if (options & pe_print_html) {
1952                 status_print("</ul>\n");
1953             }
1954         } else {
1955             child_text = crm_strdup_printf("%s  ", pre_text);
1956             print_bundle_replica(replica, child_text, options, print_data);
1957         }
1958         free(child_text);
1959 
1960         if (options & pe_print_html) {
1961             status_print("</li>\n");
1962         }
1963     }
1964     if (options & pe_print_html) {
1965         status_print("</ul>\n");
1966     }
1967 }
1968 
1969 static void
1970 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1971 {
1972     if (replica == NULL) {
1973         return;
1974     }
1975 
1976     if (replica->node) {
1977         free(replica->node);
1978         replica->node = NULL;
1979     }
1980 
1981     if (replica->ip) {
1982         free_xml(replica->ip->xml);
1983         replica->ip->xml = NULL;
1984         replica->ip->fns->free(replica->ip);
1985         replica->ip = NULL;
1986     }
1987     if (replica->container) {
1988         free_xml(replica->container->xml);
1989         replica->container->xml = NULL;
1990         replica->container->fns->free(replica->container);
1991         replica->container = NULL;
1992     }
1993     if (replica->remote) {
1994         free_xml(replica->remote->xml);
1995         replica->remote->xml = NULL;
1996         replica->remote->fns->free(replica->remote);
1997         replica->remote = NULL;
1998     }
1999     free(replica->ipaddr);
2000     free(replica);
2001 }
2002 
2003 void
2004 pe__free_bundle(pe_resource_t *rsc)
     /*  */
2005 {
2006     pe__bundle_variant_data_t *bundle_data = NULL;
2007     CRM_CHECK(rsc != NULL, return);
2008 
2009     get_bundle_variant_data(bundle_data, rsc);
2010     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
2011 
2012     free(bundle_data->prefix);
2013     free(bundle_data->image);
2014     free(bundle_data->control_port);
2015     free(bundle_data->host_network);
2016     free(bundle_data->host_netmask);
2017     free(bundle_data->ip_range_start);
2018     free(bundle_data->container_network);
2019     free(bundle_data->launcher_options);
2020     free(bundle_data->container_command);
2021     free(bundle_data->container_host_options);
2022 
2023     g_list_free_full(bundle_data->replicas,
2024                      (GDestroyNotify) free_bundle_replica);
2025     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
2026     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
2027     g_list_free(rsc->children);
2028 
2029     if(bundle_data->child) {
2030         free_xml(bundle_data->child->xml);
2031         bundle_data->child->xml = NULL;
2032         bundle_data->child->fns->free(bundle_data->child);
2033     }
2034     common_free(rsc);
2035 }
2036 
2037 enum rsc_role_e
2038 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
2039 {
2040     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
2041     return container_role;
2042 }
2043 
2044 /*!
2045  * \brief Get the number of configured replicas in a bundle
2046  *
2047  * \param[in] rsc  Bundle resource
2048  *
2049  * \return Number of configured replicas, or 0 on error
2050  */
2051 int
2052 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
2053 {
2054     if ((rsc == NULL) || (rsc->variant != pe_container)) {
2055         return 0;
2056     } else {
2057         pe__bundle_variant_data_t *bundle_data = NULL;
2058 
2059         get_bundle_variant_data(bundle_data, rsc);
2060         return bundle_data->nreplicas;
2061     }
2062 }
2063 
2064 void
2065 pe__count_bundle(pe_resource_t *rsc)
     /*  */
2066 {
2067     pe__bundle_variant_data_t *bundle_data = NULL;
2068 
2069     get_bundle_variant_data(bundle_data, rsc);
2070     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2071         pe__bundle_replica_t *replica = item->data;
2072 
2073         if (replica->ip) {
2074             replica->ip->fns->count(replica->ip);
2075         }
2076         if (replica->child) {
2077             replica->child->fns->count(replica->child);
2078         }
2079         if (replica->container) {
2080             replica->container->fns->count(replica->container);
2081         }
2082         if (replica->remote) {
2083             replica->remote->fns->count(replica->remote);
2084         }
2085     }
2086 }
2087 
2088 gboolean
2089 pe__bundle_is_filtered(pe_resource_t *rsc, GListPtr only_rsc, gboolean check_parent)
     /*  */
2090 {
2091     gboolean passes = FALSE;
2092     pe__bundle_variant_data_t *bundle_data = NULL;
2093 
2094     if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc))) {
2095         passes = TRUE;
2096     } else {
2097         get_bundle_variant_data(bundle_data, rsc);
2098 
2099         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2100             pe__bundle_replica_t *replica = gIter->data;
2101 
2102             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2103                 passes = TRUE;
2104                 break;
2105             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2106                 passes = TRUE;
2107                 break;
2108             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2109                 passes = TRUE;
2110                 break;
2111             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2112                 passes = TRUE;
2113                 break;
2114             }
2115         }
2116     }
2117 
2118     return !passes;
2119 }