lib/pengine/bundle.c

/* */
This source file includes following definitions.
next_ip
allocate_ip
create_resource
valid_network
create_ip_resource
create_docker_resource
create_podman_resource
create_rkt_resource
disallow_node
create_remote_resource
create_container
mount_add
mount_free
port_free
replica_for_remote
pe__bundle_needs_remote_name
pe__add_bundle_remote_name
pe__unpack_bundle
replica_resource_active
pe__bundle_active
pe__find_bundle_replica
print_rsc_in_list
container_agent_str
bundle_print_xml
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_html
PCMK__OUTPUT_ARGS
pe__bundle_replica_output_text
PCMK__OUTPUT_ARGS
print_bundle_replica
pe__print_bundle
free_bundle_replica
pe__free_bundle
pe__bundle_resource_state
pe_bundle_replicas
pe__count_bundle
pe__bundle_is_filtered
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU Lesser General Public License
   7  * version 2.1 or later (LGPLv2.1+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <ctype.h>
  13 
  14 #include <crm/pengine/rules.h>
  15 #include <crm/pengine/status.h>
  16 #include <crm/pengine/internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/common/xml_internal.h>
  19 #include <pe_status_private.h>
  20 
  21 #define PE__VARIANT_BUNDLE 1
  22 #include "./variant.h"
  23 
  24 static char *
  25 next_ip(const char *last_ip)
     /*  */
  26 {
  27     unsigned int oct1 = 0;
  28     unsigned int oct2 = 0;
  29     unsigned int oct3 = 0;
  30     unsigned int oct4 = 0;
  31     int rc = sscanf(last_ip, "%u.%u.%u.%u", &oct1, &oct2, &oct3, &oct4);
  32 
  33     if (rc != 4) {
  34         /*@ TODO check for IPv6 */
  35         return NULL;
  36 
  37     } else if (oct3 > 253) {
  38         return NULL;
  39 
  40     } else if (oct4 > 253) {
  41         ++oct3;
  42         oct4 = 1;
  43 
  44     } else {
  45         ++oct4;
  46     }
  47 
  48     return crm_strdup_printf("%u.%u.%u.%u", oct1, oct2, oct3, oct4);
  49 }
  50 
  51 static int
  52 allocate_ip(pe__bundle_variant_data_t *data, pe__bundle_replica_t *replica,
     /*  */
  53             char *buffer, int max)
  54 {
  55     if(data->ip_range_start == NULL) {
  56         return 0;
  57 
  58     } else if(data->ip_last) {
  59         replica->ipaddr = next_ip(data->ip_last);
  60 
  61     } else {
  62         replica->ipaddr = strdup(data->ip_range_start);
  63     }
  64 
  65     data->ip_last = replica->ipaddr;
  66     switch (data->agent_type) {
  67         case PE__CONTAINER_AGENT_DOCKER:
  68         case PE__CONTAINER_AGENT_PODMAN:
  69             if (data->add_host) {
  70                 return snprintf(buffer, max, " --add-host=%s-%d:%s",
  71                                 data->prefix, replica->offset,
  72                                 replica->ipaddr);
  73             }
  74         case PE__CONTAINER_AGENT_RKT:
  75             return snprintf(buffer, max, " --hosts-entry=%s=%s-%d",
  76                             replica->ipaddr, data->prefix, replica->offset);
  77         default: // PE__CONTAINER_AGENT_UNKNOWN
  78             break;
  79     }
  80     return 0;
  81 }
  82 
  83 static xmlNode *
  84 create_resource(const char *name, const char *provider, const char *kind)
     /*  */
  85 {
  86     xmlNode *rsc = create_xml_node(NULL, XML_CIB_TAG_RESOURCE);
  87 
  88     crm_xml_add(rsc, XML_ATTR_ID, name);
  89     crm_xml_add(rsc, XML_AGENT_ATTR_CLASS, PCMK_RESOURCE_CLASS_OCF);
  90     crm_xml_add(rsc, XML_AGENT_ATTR_PROVIDER, provider);
  91     crm_xml_add(rsc, XML_ATTR_TYPE, kind);
  92 
  93     return rsc;
  94 }
  95 
  96 /*!
  97  * \internal
  98  * \brief Check whether cluster can manage resource inside container
  99  *
 100  * \param[in] data  Container variant data
 101  *
 102  * \return TRUE if networking configuration is acceptable, FALSE otherwise
 103  *
 104  * \note The resource is manageable if an IP range or control port has been
 105  *       specified. If a control port is used without an IP range, replicas per
 106  *       host must be 1.
 107  */
 108 static bool
 109 valid_network(pe__bundle_variant_data_t *data)
     /*  */
 110 {
 111     if(data->ip_range_start) {
 112         return TRUE;
 113     }
 114     if(data->control_port) {
 115         if(data->nreplicas_per_host > 1) {
 116             pe_err("Specifying the 'control-port' for %s requires 'replicas-per-host=1'", data->prefix);
 117             data->nreplicas_per_host = 1;
 118             // @TODO to be sure: pe__clear_resource_flags(rsc, pe_rsc_unique);
 119         }
 120         return TRUE;
 121     }
 122     return FALSE;
 123 }
 124 
 125 static bool
 126 create_ip_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 127                    pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 128 {
 129     if(data->ip_range_start) {
 130         char *id = NULL;
 131         xmlNode *xml_ip = NULL;
 132         xmlNode *xml_obj = NULL;
 133 
 134         id = crm_strdup_printf("%s-ip-%s", data->prefix, replica->ipaddr);
 135         crm_xml_sanitize_id(id);
 136         xml_ip = create_resource(id, "heartbeat", "IPaddr2");
 137         free(id);
 138 
 139         xml_obj = create_xml_node(xml_ip, XML_TAG_ATTR_SETS);
 140         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 141                        data->prefix, replica->offset);
 142 
 143         crm_create_nvpair_xml(xml_obj, NULL, "ip", replica->ipaddr);
 144         if(data->host_network) {
 145             crm_create_nvpair_xml(xml_obj, NULL, "nic", data->host_network);
 146         }
 147 
 148         if(data->host_netmask) {
 149             crm_create_nvpair_xml(xml_obj, NULL,
 150                                   "cidr_netmask", data->host_netmask);
 151 
 152         } else {
 153             crm_create_nvpair_xml(xml_obj, NULL, "cidr_netmask", "32");
 154         }
 155 
 156         xml_obj = create_xml_node(xml_ip, "operations");
 157         crm_create_op_xml(xml_obj, ID(xml_ip), "monitor", "60s", NULL);
 158 
 159         // TODO: Other ops? Timeouts and intervals from underlying resource?
 160 
 161         if (!common_unpack(xml_ip, &replica->ip, parent, data_set)) {
 162             return FALSE;
 163         }
 164 
 165         parent->children = g_list_append(parent->children, replica->ip);
 166     }
 167     return TRUE;
 168 }
 169 
 170 static bool
 171 create_docker_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 172                        pe__bundle_replica_t *replica,
 173                        pe_working_set_t *data_set)
 174 {
 175         int offset = 0, max = 4096;
 176         char *buffer = calloc(1, max+1);
 177 
 178         int doffset = 0, dmax = 1024;
 179         char *dbuffer = calloc(1, dmax+1);
 180 
 181         char *id = NULL;
 182         xmlNode *xml_container = NULL;
 183         xmlNode *xml_obj = NULL;
 184 
 185         id = crm_strdup_printf("%s-docker-%d", data->prefix, replica->offset);
 186         crm_xml_sanitize_id(id);
 187         xml_container = create_resource(id, "heartbeat",
 188                                         PE__CONTAINER_AGENT_DOCKER_S);
 189         free(id);
 190 
 191         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 192         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 193                        data->prefix, replica->offset);
 194 
 195         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 196         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 197         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 198         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 199 
 200         offset += snprintf(buffer+offset, max-offset, " --restart=no");
 201 
 202         /* Set a container hostname only if we have an IP to map it to.
 203          * The user can set -h or --uts=host themselves if they want a nicer
 204          * name for logs, but this makes applications happy who need their
 205          * hostname to match the IP they bind to.
 206          */
 207         if (data->ip_range_start != NULL) {
 208             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 209                                data->prefix, replica->offset);
 210         }
 211 
 212         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 213 
 214         if (data->container_network) {
 215 #if 0
 216             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 217                                replica->ipaddr);
 218 #endif
 219             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 220                                data->container_network);
 221         }
 222 
 223         if(data->control_port) {
 224             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 225         } else {
 226             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 227         }
 228 
 229         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 230             pe__bundle_mount_t *mount = pIter->data;
 231 
 232             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 233                 char *source = crm_strdup_printf(
 234                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 235 
 236                 if(doffset > 0) {
 237                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 238                 }
 239                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 240                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 241                 free(source);
 242 
 243             } else {
 244                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 245             }
 246             if(mount->options) {
 247                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 248             }
 249         }
 250 
 251         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 252             pe__bundle_port_t *port = pIter->data;
 253 
 254             if (replica->ipaddr) {
 255                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 256                                    replica->ipaddr, port->source,
 257                                    port->target);
 258             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 259                 // No need to do port mapping if net=host
 260                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 261             }
 262         }
 263 
 264         if (data->launcher_options) {
 265             offset += snprintf(buffer+offset, max-offset, " %s",
 266                                data->launcher_options);
 267         }
 268 
 269         if (data->container_host_options) {
 270             offset += snprintf(buffer + offset, max - offset, " %s",
 271                                data->container_host_options);
 272         }
 273 
 274         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 275         free(buffer);
 276 
 277         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 278         free(dbuffer);
 279 
 280         if (replica->child) {
 281             if (data->container_command) {
 282                 crm_create_nvpair_xml(xml_obj, NULL,
 283                                       "run_cmd", data->container_command);
 284             } else {
 285                 crm_create_nvpair_xml(xml_obj, NULL,
 286                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 287             }
 288 
 289             /* TODO: Allow users to specify their own?
 290              *
 291              * We just want to know if the container is alive, we'll
 292              * monitor the child independently
 293              */
 294             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 295 #if 0
 296         /* @TODO Consider supporting the use case where we can start and stop
 297          * resources, but not proxy local commands (such as setting node
 298          * attributes), by running the local executor in stand-alone mode.
 299          * However, this would probably be better done via ACLs as with other
 300          * Pacemaker Remote nodes.
 301          */
 302         } else if ((child != NULL) && data->untrusted) {
 303             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 304                                   CRM_DAEMON_DIR "/pacemaker-execd");
 305             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 306                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 307 #endif
 308         } else {
 309             if (data->container_command) {
 310                 crm_create_nvpair_xml(xml_obj, NULL,
 311                                       "run_cmd", data->container_command);
 312             }
 313 
 314             /* TODO: Allow users to specify their own?
 315              *
 316              * We don't know what's in the container, so we just want
 317              * to know if it is alive
 318              */
 319             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 320         }
 321 
 322 
 323         xml_obj = create_xml_node(xml_container, "operations");
 324         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 325 
 326         // TODO: Other ops? Timeouts and intervals from underlying resource?
 327         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 328             return FALSE;
 329         }
 330         parent->children = g_list_append(parent->children, replica->container);
 331         return TRUE;
 332 }
 333 
 334 static bool
 335 create_podman_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 336                        pe__bundle_replica_t *replica,
 337                        pe_working_set_t *data_set)
 338 {
 339         int offset = 0, max = 4096;
 340         char *buffer = calloc(1, max+1);
 341 
 342         int doffset = 0, dmax = 1024;
 343         char *dbuffer = calloc(1, dmax+1);
 344 
 345         char *id = NULL;
 346         xmlNode *xml_container = NULL;
 347         xmlNode *xml_obj = NULL;
 348 
 349         id = crm_strdup_printf("%s-podman-%d", data->prefix, replica->offset);
 350         crm_xml_sanitize_id(id);
 351         xml_container = create_resource(id, "heartbeat",
 352                                         PE__CONTAINER_AGENT_PODMAN_S);
 353         free(id);
 354 
 355         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 356         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 357                        data->prefix, replica->offset);
 358 
 359         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 360         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", XML_BOOLEAN_TRUE);
 361         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", XML_BOOLEAN_FALSE);
 362         crm_create_nvpair_xml(xml_obj, NULL, "reuse", XML_BOOLEAN_FALSE);
 363 
 364         // FIXME: (bandini 2018-08) podman has no restart policies
 365         //offset += snprintf(buffer+offset, max-offset, " --restart=no");
 366 
 367         /* Set a container hostname only if we have an IP to map it to.
 368          * The user can set -h or --uts=host themselves if they want a nicer
 369          * name for logs, but this makes applications happy who need their
 370          * hostname to match the IP they bind to.
 371          */
 372         if (data->ip_range_start != NULL) {
 373             offset += snprintf(buffer+offset, max-offset, " -h %s-%d",
 374                                data->prefix, replica->offset);
 375         }
 376 
 377         offset += snprintf(buffer+offset, max-offset, " -e PCMK_stderr=1");
 378 
 379         if (data->container_network) {
 380 #if 0
 381             // podman has no support for --link-local-ip
 382             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 383                                replica->ipaddr);
 384 #endif
 385             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 386                                data->container_network);
 387         }
 388 
 389         if(data->control_port) {
 390             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%s", data->control_port);
 391         } else {
 392             offset += snprintf(buffer+offset, max-offset, " -e PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 393         }
 394 
 395         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 396             pe__bundle_mount_t *mount = pIter->data;
 397 
 398             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 399                 char *source = crm_strdup_printf(
 400                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 401 
 402                 if(doffset > 0) {
 403                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 404                 }
 405                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 406                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", source, mount->target);
 407                 free(source);
 408 
 409             } else {
 410                 offset += snprintf(buffer+offset, max-offset, " -v %s:%s", mount->source, mount->target);
 411             }
 412             if(mount->options) {
 413                 offset += snprintf(buffer+offset, max-offset, ":%s", mount->options);
 414             }
 415         }
 416 
 417         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 418             pe__bundle_port_t *port = pIter->data;
 419 
 420             if (replica->ipaddr) {
 421                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s:%s",
 422                                    replica->ipaddr, port->source,
 423                                    port->target);
 424             } else if(!pcmk__str_eq(data->container_network, "host", pcmk__str_casei)) {
 425                 // No need to do port mapping if net=host
 426                 offset += snprintf(buffer+offset, max-offset, " -p %s:%s", port->source, port->target);
 427             }
 428         }
 429 
 430         if (data->launcher_options) {
 431             offset += snprintf(buffer+offset, max-offset, " %s",
 432                                data->launcher_options);
 433         }
 434 
 435         if (data->container_host_options) {
 436             offset += snprintf(buffer + offset, max - offset, " %s",
 437                                data->container_host_options);
 438         }
 439 
 440         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 441         free(buffer);
 442 
 443         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 444         free(dbuffer);
 445 
 446         if (replica->child) {
 447             if (data->container_command) {
 448                 crm_create_nvpair_xml(xml_obj, NULL,
 449                                       "run_cmd", data->container_command);
 450             } else {
 451                 crm_create_nvpair_xml(xml_obj, NULL,
 452                                       "run_cmd", SBIN_DIR "/pacemaker-remoted");
 453             }
 454 
 455             /* TODO: Allow users to specify their own?
 456              *
 457              * We just want to know if the container is alive, we'll
 458              * monitor the child independently
 459              */
 460             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 461 #if 0
 462         /* @TODO Consider supporting the use case where we can start and stop
 463          * resources, but not proxy local commands (such as setting node
 464          * attributes), by running the local executor in stand-alone mode.
 465          * However, this would probably be better done via ACLs as with other
 466          * Pacemaker Remote nodes.
 467          */
 468         } else if ((child != NULL) && data->untrusted) {
 469             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 470                                   CRM_DAEMON_DIR "/pacemaker-execd");
 471             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 472                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 473 #endif
 474         } else {
 475             if (data->container_command) {
 476                 crm_create_nvpair_xml(xml_obj, NULL,
 477                                       "run_cmd", data->container_command);
 478             }
 479 
 480             /* TODO: Allow users to specify their own?
 481              *
 482              * We don't know what's in the container, so we just want
 483              * to know if it is alive
 484              */
 485             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 486         }
 487 
 488 
 489         xml_obj = create_xml_node(xml_container, "operations");
 490         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 491 
 492         // TODO: Other ops? Timeouts and intervals from underlying resource?
 493         if (!common_unpack(xml_container, &replica->container, parent,
 494                            data_set)) {
 495             return FALSE;
 496         }
 497         parent->children = g_list_append(parent->children, replica->container);
 498         return TRUE;
 499 }
 500 
 501 static bool
 502 create_rkt_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 503                     pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 504 {
 505         int offset = 0, max = 4096;
 506         char *buffer = calloc(1, max+1);
 507 
 508         int doffset = 0, dmax = 1024;
 509         char *dbuffer = calloc(1, dmax+1);
 510 
 511         char *id = NULL;
 512         xmlNode *xml_container = NULL;
 513         xmlNode *xml_obj = NULL;
 514 
 515         int volid = 0;
 516 
 517         id = crm_strdup_printf("%s-rkt-%d", data->prefix, replica->offset);
 518         crm_xml_sanitize_id(id);
 519         xml_container = create_resource(id, "heartbeat",
 520                                         PE__CONTAINER_AGENT_RKT_S);
 521         free(id);
 522 
 523         xml_obj = create_xml_node(xml_container, XML_TAG_ATTR_SETS);
 524         crm_xml_set_id(xml_obj, "%s-attributes-%d",
 525                        data->prefix, replica->offset);
 526 
 527         crm_create_nvpair_xml(xml_obj, NULL, "image", data->image);
 528         crm_create_nvpair_xml(xml_obj, NULL, "allow_pull", "true");
 529         crm_create_nvpair_xml(xml_obj, NULL, "force_kill", "false");
 530         crm_create_nvpair_xml(xml_obj, NULL, "reuse", "false");
 531 
 532         /* Set a container hostname only if we have an IP to map it to.
 533          * The user can set -h or --uts=host themselves if they want a nicer
 534          * name for logs, but this makes applications happy who need their
 535          * hostname to match the IP they bind to.
 536          */
 537         if (data->ip_range_start != NULL) {
 538             offset += snprintf(buffer+offset, max-offset, " --hostname=%s-%d",
 539                                data->prefix, replica->offset);
 540         }
 541 
 542         offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_stderr=1");
 543 
 544         if (data->container_network) {
 545 #if 0
 546             offset += snprintf(buffer+offset, max-offset, " --link-local-ip=%s",
 547                                replica->ipaddr);
 548 #endif
 549             offset += snprintf(buffer+offset, max-offset, " --net=%s",
 550                                data->container_network);
 551         }
 552 
 553         if(data->control_port) {
 554             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%s", data->control_port);
 555         } else {
 556             offset += snprintf(buffer+offset, max-offset, " --environment=PCMK_remote_port=%d", DEFAULT_REMOTE_PORT);
 557         }
 558 
 559         for(GList *pIter = data->mounts; pIter != NULL; pIter = pIter->next) {
 560             pe__bundle_mount_t *mount = pIter->data;
 561 
 562             if (pcmk_is_set(mount->flags, pe__bundle_mount_subdir)) {
 563                 char *source = crm_strdup_printf(
 564                     "%s/%s-%d", mount->source, data->prefix, replica->offset);
 565 
 566                 if(doffset > 0) {
 567                     doffset += snprintf(dbuffer+doffset, dmax-doffset, ",");
 568                 }
 569                 doffset += snprintf(dbuffer+doffset, dmax-doffset, "%s", source);
 570                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, source);
 571                 if(mount->options) {
 572                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 573                 }
 574                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 575                 free(source);
 576 
 577             } else {
 578                 offset += snprintf(buffer+offset, max-offset, " --volume vol%d,kind=host,source=%s", volid, mount->source);
 579                 if(mount->options) {
 580                     offset += snprintf(buffer+offset, max-offset, ",%s", mount->options);
 581                 }
 582                 offset += snprintf(buffer+offset, max-offset, " --mount volume=vol%d,target=%s", volid, mount->target);
 583             }
 584             volid++;
 585         }
 586 
 587         for(GList *pIter = data->ports; pIter != NULL; pIter = pIter->next) {
 588             pe__bundle_port_t *port = pIter->data;
 589 
 590             if (replica->ipaddr) {
 591                 offset += snprintf(buffer+offset, max-offset,
 592                                    " --port=%s:%s:%s", port->target,
 593                                    replica->ipaddr, port->source);
 594             } else {
 595                 offset += snprintf(buffer+offset, max-offset, " --port=%s:%s", port->target, port->source);
 596             }
 597         }
 598 
 599         if (data->launcher_options) {
 600             offset += snprintf(buffer+offset, max-offset, " %s",
 601                                data->launcher_options);
 602         }
 603 
 604         if (data->container_host_options) {
 605             offset += snprintf(buffer + offset, max - offset, " %s",
 606                                data->container_host_options);
 607         }
 608 
 609         crm_create_nvpair_xml(xml_obj, NULL, "run_opts", buffer);
 610         free(buffer);
 611 
 612         crm_create_nvpair_xml(xml_obj, NULL, "mount_points", dbuffer);
 613         free(dbuffer);
 614 
 615         if (replica->child) {
 616             if (data->container_command) {
 617                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 618                                       data->container_command);
 619             } else {
 620                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 621                                       SBIN_DIR "/pacemaker-remoted");
 622             }
 623 
 624             /* TODO: Allow users to specify their own?
 625              *
 626              * We just want to know if the container is alive, we'll
 627              * monitor the child independently
 628              */
 629             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 630 #if 0
 631         /* @TODO Consider supporting the use case where we can start and stop
 632          * resources, but not proxy local commands (such as setting node
 633          * attributes), by running the local executor in stand-alone mode.
 634          * However, this would probably be better done via ACLs as with other
 635          * Pacemaker Remote nodes.
 636          */
 637         } else if ((child != NULL) && data->untrusted) {
 638             crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 639                                   CRM_DAEMON_DIR "/pacemaker-execd");
 640             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd",
 641                                   CRM_DAEMON_DIR "/pacemaker/cts-exec-helper -c poke");
 642 #endif
 643         } else {
 644             if (data->container_command) {
 645                 crm_create_nvpair_xml(xml_obj, NULL, "run_cmd",
 646                                       data->container_command);
 647             }
 648 
 649             /* TODO: Allow users to specify their own?
 650              *
 651              * We don't know what's in the container, so we just want
 652              * to know if it is alive
 653              */
 654             crm_create_nvpair_xml(xml_obj, NULL, "monitor_cmd", "/bin/true");
 655         }
 656 
 657 
 658         xml_obj = create_xml_node(xml_container, "operations");
 659         crm_create_op_xml(xml_obj, ID(xml_container), "monitor", "60s", NULL);
 660 
 661         // TODO: Other ops? Timeouts and intervals from underlying resource?
 662 
 663         if (!common_unpack(xml_container, &replica->container, parent, data_set)) {
 664             return FALSE;
 665         }
 666         parent->children = g_list_append(parent->children, replica->container);
 667         return TRUE;
 668 }
 669 
 670 /*!
 671  * \brief Ban a node from a resource's (and its children's) allowed nodes list
 672  *
 673  * \param[in,out] rsc    Resource to modify
 674  * \param[in]     uname  Name of node to ban
 675  */
 676 static void
 677 disallow_node(pe_resource_t *rsc, const char *uname)
     /*  */
 678 {
 679     gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
 680 
 681     if (match) {
 682         ((pe_node_t *) match)->weight = -INFINITY;
 683         ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never;
 684     }
 685     if (rsc->children) {
 686         GList *child;
 687 
 688         for (child = rsc->children; child != NULL; child = child->next) {
 689             disallow_node((pe_resource_t *) (child->data), uname);
 690         }
 691     }
 692 }
 693 
 694 static bool
 695 create_remote_resource(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 696                        pe__bundle_replica_t *replica,
 697                        pe_working_set_t *data_set)
 698 {
 699     if (replica->child && valid_network(data)) {
 700         GHashTableIter gIter;
 701         GList *rsc_iter = NULL;
 702         pe_node_t *node = NULL;
 703         xmlNode *xml_remote = NULL;
 704         char *id = crm_strdup_printf("%s-%d", data->prefix, replica->offset);
 705         char *port_s = NULL;
 706         const char *uname = NULL;
 707         const char *connect_name = NULL;
 708 
 709         if (pe_find_resource(data_set->resources, id) != NULL) {
 710             free(id);
 711             // The biggest hammer we have
 712             id = crm_strdup_printf("pcmk-internal-%s-remote-%d",
 713                                    replica->child->id, replica->offset);
 714             //@TODO return false instead of asserting?
 715             CRM_ASSERT(pe_find_resource(data_set->resources, id) == NULL);
 716         }
 717 
 718         /* REMOTE_CONTAINER_HACK: Using "#uname" as the server name when the
 719          * connection does not have its own IP is a magic string that we use to
 720          * support nested remotes (i.e. a bundle running on a remote node).
 721          */
 722         connect_name = (replica->ipaddr? replica->ipaddr : "#uname");
 723 
 724         if (data->control_port == NULL) {
 725             port_s = pcmk__itoa(DEFAULT_REMOTE_PORT);
 726         }
 727 
 728         /* This sets replica->container as replica->remote's container, which is
 729          * similar to what happens with guest nodes. This is how the scheduler
 730          * knows that the bundle node is fenced by recovering the container, and
 731          * that remote should be ordered relative to the container.
 732          */
 733         xml_remote = pe_create_remote_xml(NULL, id, replica->container->id,
 734                                           NULL, NULL, NULL,
 735                                           connect_name, (data->control_port?
 736                                           data->control_port : port_s));
 737         free(port_s);
 738 
 739         /* Abandon our created ID, and pull the copy from the XML, because we
 740          * need something that will get freed during data set cleanup to use as
 741          * the node ID and uname.
 742          */
 743         free(id);
 744         id = NULL;
 745         uname = ID(xml_remote);
 746 
 747         /* Ensure a node has been created for the guest (it may have already
 748          * been, if it has a permanent node attribute), and ensure its weight is
 749          * -INFINITY so no other resources can run on it.
 750          */
 751         node = pe_find_node(data_set->nodes, uname);
 752         if (node == NULL) {
 753             node = pe_create_node(uname, uname, "remote", "-INFINITY",
 754                                   data_set);
 755         } else {
 756             node->weight = -INFINITY;
 757         }
 758         node->rsc_discover_mode = pe_discover_never;
 759 
 760         /* unpack_remote_nodes() ensures that each remote node and guest node
 761          * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
 762          * Unfortunately, a bundle has to be mostly unpacked before it's obvious
 763          * what nodes will be needed, so we do it just above.
 764          *
 765          * Worse, that means that the node may have been utilized while
 766          * unpacking other resources, without our weight correction. The most
 767          * likely place for this to happen is when common_unpack() calls
 768          * resource_location() to set a default score in symmetric clusters.
 769          * This adds a node *copy* to each resource's allowed nodes, and these
 770          * copies will have the wrong weight.
 771          *
 772          * As a hacky workaround, fix those copies here.
 773          *
 774          * @TODO Possible alternative: ensure bundles are unpacked before other
 775          * resources, so the weight is correct before any copies are made.
 776          */
 777         for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
 778             disallow_node((pe_resource_t *) (rsc_iter->data), uname);
 779         }
 780 
 781         replica->node = pe__copy_node(node);
 782         replica->node->weight = 500;
 783         replica->node->rsc_discover_mode = pe_discover_exclusive;
 784 
 785         /* Ensure the node shows up as allowed and with the correct discovery set */
 786         if (replica->child->allowed_nodes != NULL) {
 787             g_hash_table_destroy(replica->child->allowed_nodes);
 788         }
 789         replica->child->allowed_nodes = pcmk__strkey_table(NULL, free);
 790         g_hash_table_insert(replica->child->allowed_nodes,
 791                             (gpointer) replica->node->details->id,
 792                             pe__copy_node(replica->node));
 793 
 794         {
 795             pe_node_t *copy = pe__copy_node(replica->node);
 796             copy->weight = -INFINITY;
 797             g_hash_table_insert(replica->child->parent->allowed_nodes,
 798                                 (gpointer) replica->node->details->id, copy);
 799         }
 800         if (!common_unpack(xml_remote, &replica->remote, parent, data_set)) {
 801             return FALSE;
 802         }
 803 
 804         g_hash_table_iter_init(&gIter, replica->remote->allowed_nodes);
 805         while (g_hash_table_iter_next(&gIter, NULL, (void **)&node)) {
 806             if (pe__is_guest_or_remote_node(node)) {
 807                 /* Remote resources can only run on 'normal' cluster node */
 808                 node->weight = -INFINITY;
 809             }
 810         }
 811 
 812         replica->node->details->remote_rsc = replica->remote;
 813 
 814         // Ensure pe__is_guest_node() functions correctly immediately
 815         replica->remote->container = replica->container;
 816 
 817         /* A bundle's #kind is closer to "container" (guest node) than the
 818          * "remote" set by pe_create_node().
 819          */
 820         g_hash_table_insert(replica->node->details->attrs,
 821                             strdup(CRM_ATTR_KIND), strdup("container"));
 822 
 823         /* One effect of this is that setup_container() will add
 824          * replica->remote to replica->container's fillers, which will make
 825          * pe__resource_contains_guest_node() true for replica->container.
 826          *
 827          * replica->child does NOT get added to replica->container's fillers.
 828          * The only noticeable effect if it did would be for its fail count to
 829          * be taken into account when checking replica->container's migration
 830          * threshold.
 831          */
 832         parent->children = g_list_append(parent->children, replica->remote);
 833     }
 834     return TRUE;
 835 }
 836 
 837 static bool
 838 create_container(pe_resource_t *parent, pe__bundle_variant_data_t *data,
     /*  */
 839                  pe__bundle_replica_t *replica, pe_working_set_t *data_set)
 840 {
 841 
 842     switch (data->agent_type) {
 843         case PE__CONTAINER_AGENT_DOCKER:
 844             if (!create_docker_resource(parent, data, replica, data_set)) {
 845                 return FALSE;
 846             }
 847             break;
 848 
 849         case PE__CONTAINER_AGENT_PODMAN:
 850             if (!create_podman_resource(parent, data, replica, data_set)) {
 851                 return FALSE;
 852             }
 853             break;
 854 
 855         case PE__CONTAINER_AGENT_RKT:
 856             if (!create_rkt_resource(parent, data, replica, data_set)) {
 857                 return FALSE;
 858             }
 859             break;
 860         default: // PE__CONTAINER_AGENT_UNKNOWN
 861             return FALSE;
 862     }
 863 
 864     if (create_ip_resource(parent, data, replica, data_set) == FALSE) {
 865         return FALSE;
 866     }
 867     if(create_remote_resource(parent, data, replica, data_set) == FALSE) {
 868         return FALSE;
 869     }
 870     if (replica->child && replica->ipaddr) {
 871         add_hash_param(replica->child->meta, "external-ip", replica->ipaddr);
 872     }
 873 
 874     if (replica->remote) {
 875         /*
 876          * Allow the remote connection resource to be allocated to a
 877          * different node than the one on which the container is active.
 878          *
 879          * This makes it possible to have Pacemaker Remote nodes running
 880          * containers with pacemaker-remoted inside in order to start
 881          * services inside those containers.
 882          */
 883         pe__set_resource_flags(replica->remote, pe_rsc_allow_remote_remotes);
 884     }
 885 
 886     return TRUE;
 887 }
 888 
 889 static void
 890 mount_add(pe__bundle_variant_data_t *bundle_data, const char *source,
     /*  */
 891           const char *target, const char *options, uint32_t flags)
 892 {
 893     pe__bundle_mount_t *mount = calloc(1, sizeof(pe__bundle_mount_t));
 894 
 895     mount->source = strdup(source);
 896     mount->target = strdup(target);
 897     if (options) {
 898         mount->options = strdup(options);
 899     }
 900     mount->flags = flags;
 901     bundle_data->mounts = g_list_append(bundle_data->mounts, mount);
 902 }
 903 
 904 static void
 905 mount_free(pe__bundle_mount_t *mount)
     /*  */
 906 {
 907     free(mount->source);
 908     free(mount->target);
 909     free(mount->options);
 910     free(mount);
 911 }
 912 
 913 static void
 914 port_free(pe__bundle_port_t *port)
     /*  */
 915 {
 916     free(port->source);
 917     free(port->target);
 918     free(port);
 919 }
 920 
 921 static pe__bundle_replica_t *
 922 replica_for_remote(pe_resource_t *remote)
     /*  */
 923 {
 924     pe_resource_t *top = remote;
 925     pe__bundle_variant_data_t *bundle_data = NULL;
 926 
 927     if (top == NULL) {
 928         return NULL;
 929     }
 930 
 931     while (top->parent != NULL) {
 932         top = top->parent;
 933     }
 934 
 935     get_bundle_variant_data(bundle_data, top);
 936     for (GList *gIter = bundle_data->replicas; gIter != NULL;
 937          gIter = gIter->next) {
 938         pe__bundle_replica_t *replica = gIter->data;
 939 
 940         if (replica->remote == remote) {
 941             return replica;
 942         }
 943     }
 944     CRM_LOG_ASSERT(FALSE);
 945     return NULL;
 946 }
 947 
 948 bool
 949 pe__bundle_needs_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 950 {
 951     const char *value;
 952     GHashTable *params = NULL;
 953 
 954     if (rsc == NULL) {
 955         return false;
 956     }
 957 
 958     // Use NULL node since pcmk__bundle_expand() uses that to set value
 959     params = pe_rsc_params(rsc, NULL, data_set);
 960     value = g_hash_table_lookup(params, XML_RSC_ATTR_REMOTE_RA_ADDR);
 961 
 962     return pcmk__str_eq(value, "#uname", pcmk__str_casei)
 963            && xml_contains_remote_node(rsc->xml);
 964 }
 965 
 966 const char *
 967 pe__add_bundle_remote_name(pe_resource_t *rsc, pe_working_set_t *data_set,
     /*  */
 968                            xmlNode *xml, const char *field)
 969 {
 970     // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 971 
 972     pe_node_t *node = NULL;
 973     pe__bundle_replica_t *replica = NULL;
 974 
 975     if (!pe__bundle_needs_remote_name(rsc, data_set)) {
 976         return NULL;
 977     }
 978 
 979     replica = replica_for_remote(rsc);
 980     if (replica == NULL) {
 981         return NULL;
 982     }
 983 
 984     node = replica->container->allocated_to;
 985     if (node == NULL) {
 986         /* If it won't be running anywhere after the
 987          * transition, go with where it's running now.
 988          */
 989         node = pe__current_node(replica->container);
 990     }
 991 
 992     if(node == NULL) {
 993         crm_trace("Cannot determine address for bundle connection %s", rsc->id);
 994         return NULL;
 995     }
 996 
 997     crm_trace("Setting address for bundle connection %s to bundle host %s",
 998               rsc->id, node->details->uname);
 999     if(xml != NULL && field != NULL) {
1000         crm_xml_add(xml, field, node->details->uname);
1001     }
1002 
1003     return node->details->uname;
1004 }
1005 
1006 #define pe__set_bundle_mount_flags(mount_xml, flags, flags_to_set) do {     \
1007         flags = pcmk__set_flags_as(__func__, __LINE__, LOG_TRACE,           \
1008                                    "Bundle mount", ID(mount_xml), flags,    \
1009                                    (flags_to_set), #flags_to_set);          \
1010     } while (0)
1011 
1012 gboolean
1013 pe__unpack_bundle(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
1014 {
1015     const char *value = NULL;
1016     xmlNode *xml_obj = NULL;
1017     xmlNode *xml_resource = NULL;
1018     pe__bundle_variant_data_t *bundle_data = NULL;
1019     bool need_log_mount = TRUE;
1020 
1021     CRM_ASSERT(rsc != NULL);
1022     pe_rsc_trace(rsc, "Processing resource %s...", rsc->id);
1023 
1024     bundle_data = calloc(1, sizeof(pe__bundle_variant_data_t));
1025     rsc->variant_opaque = bundle_data;
1026     bundle_data->prefix = strdup(rsc->id);
1027 
1028     xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_DOCKER_S);
1029     if (xml_obj != NULL) {
1030         bundle_data->agent_type = PE__CONTAINER_AGENT_DOCKER;
1031     } else {
1032         xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_RKT_S);
1033         if (xml_obj != NULL) {
1034             bundle_data->agent_type = PE__CONTAINER_AGENT_RKT;
1035         } else {
1036             xml_obj = first_named_child(rsc->xml, PE__CONTAINER_AGENT_PODMAN_S);
1037             if (xml_obj != NULL) {
1038                 bundle_data->agent_type = PE__CONTAINER_AGENT_PODMAN;
1039             } else {
1040                 return FALSE;
1041             }
1042         }
1043     }
1044 
1045     // Use 0 for default, minimum, and invalid promoted-max
1046     value = crm_element_value(xml_obj, XML_RSC_ATTR_PROMOTED_MAX);
1047     if (value == NULL) {
1048         // @COMPAT deprecated since 2.0.0
1049         value = crm_element_value(xml_obj, "masters");
1050     }
1051     pcmk__scan_min_int(value, &bundle_data->promoted_max, 0);
1052 
1053     // Default replicas to promoted-max if it was specified and 1 otherwise
1054     value = crm_element_value(xml_obj, "replicas");
1055     if ((value == NULL) && (bundle_data->promoted_max > 0)) {
1056         bundle_data->nreplicas = bundle_data->promoted_max;
1057     } else {
1058         pcmk__scan_min_int(value, &bundle_data->nreplicas, 1);
1059     }
1060 
1061     /*
1062      * Communication between containers on the same host via the
1063      * floating IPs only works if the container is started with:
1064      *   --userland-proxy=false --ip-masq=false
1065      */
1066     value = crm_element_value(xml_obj, "replicas-per-host");
1067     pcmk__scan_min_int(value, &bundle_data->nreplicas_per_host, 1);
1068     if (bundle_data->nreplicas_per_host == 1) {
1069         pe__clear_resource_flags(rsc, pe_rsc_unique);
1070     }
1071 
1072     bundle_data->container_command = crm_element_value_copy(xml_obj, "run-command");
1073     bundle_data->launcher_options = crm_element_value_copy(xml_obj, "options");
1074     bundle_data->image = crm_element_value_copy(xml_obj, "image");
1075     bundle_data->container_network = crm_element_value_copy(xml_obj, "network");
1076 
1077     xml_obj = first_named_child(rsc->xml, "network");
1078     if(xml_obj) {
1079 
1080         bundle_data->ip_range_start = crm_element_value_copy(xml_obj, "ip-range-start");
1081         bundle_data->host_netmask = crm_element_value_copy(xml_obj, "host-netmask");
1082         bundle_data->host_network = crm_element_value_copy(xml_obj, "host-interface");
1083         bundle_data->control_port = crm_element_value_copy(xml_obj, "control-port");
1084         value = crm_element_value(xml_obj, "add-host");
1085         if (crm_str_to_boolean(value, &bundle_data->add_host) != 1) {
1086             bundle_data->add_host = TRUE;
1087         }
1088 
1089         for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1090              xml_child = pcmk__xe_next(xml_child)) {
1091 
1092             pe__bundle_port_t *port = calloc(1, sizeof(pe__bundle_port_t));
1093             port->source = crm_element_value_copy(xml_child, "port");
1094 
1095             if(port->source == NULL) {
1096                 port->source = crm_element_value_copy(xml_child, "range");
1097             } else {
1098                 port->target = crm_element_value_copy(xml_child, "internal-port");
1099             }
1100 
1101             if(port->source != NULL && strlen(port->source) > 0) {
1102                 if(port->target == NULL) {
1103                     port->target = strdup(port->source);
1104                 }
1105                 bundle_data->ports = g_list_append(bundle_data->ports, port);
1106 
1107             } else {
1108                 pe_err("Invalid port directive %s", ID(xml_child));
1109                 port_free(port);
1110             }
1111         }
1112     }
1113 
1114     xml_obj = first_named_child(rsc->xml, "storage");
1115     for (xmlNode *xml_child = pcmk__xe_first_child(xml_obj); xml_child != NULL;
1116          xml_child = pcmk__xe_next(xml_child)) {
1117 
1118         const char *source = crm_element_value(xml_child, "source-dir");
1119         const char *target = crm_element_value(xml_child, "target-dir");
1120         const char *options = crm_element_value(xml_child, "options");
1121         int flags = pe__bundle_mount_none;
1122 
1123         if (source == NULL) {
1124             source = crm_element_value(xml_child, "source-dir-root");
1125             pe__set_bundle_mount_flags(xml_child, flags,
1126                                        pe__bundle_mount_subdir);
1127         }
1128 
1129         if (source && target) {
1130             mount_add(bundle_data, source, target, options, flags);
1131             if (strcmp(target, "/var/log") == 0) {
1132                 need_log_mount = FALSE;
1133             }
1134         } else {
1135             pe_err("Invalid mount directive %s", ID(xml_child));
1136         }
1137     }
1138 
1139     xml_obj = first_named_child(rsc->xml, "primitive");
1140     if (xml_obj && valid_network(bundle_data)) {
1141         char *value = NULL;
1142         xmlNode *xml_set = NULL;
1143 
1144         xml_resource = create_xml_node(NULL, XML_CIB_TAG_INCARNATION);
1145 
1146         /* @COMPAT We no longer use the <master> tag, but we need to keep it as
1147          * part of the resource name, so that bundles don't restart in a rolling
1148          * upgrade. (It also avoids needing to change regression tests.)
1149          */
1150         crm_xml_set_id(xml_resource, "%s-%s", bundle_data->prefix,
1151                       (bundle_data->promoted_max? "master"
1152                       : (const char *)xml_resource->name));
1153 
1154         xml_set = create_xml_node(xml_resource, XML_TAG_META_SETS);
1155         crm_xml_set_id(xml_set, "%s-%s-meta", bundle_data->prefix, xml_resource->name);
1156 
1157         crm_create_nvpair_xml(xml_set, NULL,
1158                               XML_RSC_ATTR_ORDERED, XML_BOOLEAN_TRUE);
1159 
1160         value = pcmk__itoa(bundle_data->nreplicas);
1161         crm_create_nvpair_xml(xml_set, NULL,
1162                               XML_RSC_ATTR_INCARNATION_MAX, value);
1163         free(value);
1164 
1165         value = pcmk__itoa(bundle_data->nreplicas_per_host);
1166         crm_create_nvpair_xml(xml_set, NULL,
1167                               XML_RSC_ATTR_INCARNATION_NODEMAX, value);
1168         free(value);
1169 
1170         crm_create_nvpair_xml(xml_set, NULL, XML_RSC_ATTR_UNIQUE,
1171                               pcmk__btoa(bundle_data->nreplicas_per_host > 1));
1172 
1173         if (bundle_data->promoted_max) {
1174             crm_create_nvpair_xml(xml_set, NULL,
1175                                   XML_RSC_ATTR_PROMOTABLE, XML_BOOLEAN_TRUE);
1176 
1177             value = pcmk__itoa(bundle_data->promoted_max);
1178             crm_create_nvpair_xml(xml_set, NULL,
1179                                   XML_RSC_ATTR_PROMOTED_MAX, value);
1180             free(value);
1181         }
1182 
1183         //crm_xml_add(xml_obj, XML_ATTR_ID, bundle_data->prefix);
1184         add_node_copy(xml_resource, xml_obj);
1185 
1186     } else if(xml_obj) {
1187         pe_err("Cannot control %s inside %s without either ip-range-start or control-port",
1188                rsc->id, ID(xml_obj));
1189         return FALSE;
1190     }
1191 
1192     if(xml_resource) {
1193         int lpc = 0;
1194         GList *childIter = NULL;
1195         pe_resource_t *new_rsc = NULL;
1196         pe__bundle_port_t *port = NULL;
1197 
1198         int offset = 0, max = 1024;
1199         char *buffer = NULL;
1200 
1201         if (common_unpack(xml_resource, &new_rsc, rsc, data_set) == FALSE) {
1202             pe_err("Failed unpacking resource %s", ID(rsc->xml));
1203             if (new_rsc != NULL && new_rsc->fns != NULL) {
1204                 new_rsc->fns->free(new_rsc);
1205             }
1206             return FALSE;
1207         }
1208 
1209         bundle_data->child = new_rsc;
1210 
1211         /* Currently, we always map the default authentication key location
1212          * into the same location inside the container.
1213          *
1214          * Ideally, we would respect the host's PCMK_authkey_location, but:
1215          * - it may be different on different nodes;
1216          * - the actual connection will do extra checking to make sure the key
1217          *   file exists and is readable, that we can't do here on the DC
1218          * - tools such as crm_resource and crm_simulate may not have the same
1219          *   environment variables as the cluster, causing operation digests to
1220          *   differ
1221          *
1222          * Always using the default location inside the container is fine,
1223          * because we control the pacemaker_remote environment, and it avoids
1224          * having to pass another environment variable to the container.
1225          *
1226          * @TODO A better solution may be to have only pacemaker_remote use the
1227          * environment variable, and have the cluster nodes use a new
1228          * cluster option for key location. This would introduce the limitation
1229          * of the location being the same on all cluster nodes, but that's
1230          * reasonable.
1231          */
1232         mount_add(bundle_data, DEFAULT_REMOTE_KEY_LOCATION,
1233                   DEFAULT_REMOTE_KEY_LOCATION, NULL, pe__bundle_mount_none);
1234 
1235         if (need_log_mount) {
1236             mount_add(bundle_data, CRM_BUNDLE_DIR, "/var/log", NULL,
1237                       pe__bundle_mount_subdir);
1238         }
1239 
1240         port = calloc(1, sizeof(pe__bundle_port_t));
1241         if(bundle_data->control_port) {
1242             port->source = strdup(bundle_data->control_port);
1243         } else {
1244             /* If we wanted to respect PCMK_remote_port, we could use
1245              * crm_default_remote_port() here and elsewhere in this file instead
1246              * of DEFAULT_REMOTE_PORT.
1247              *
1248              * However, it gains nothing, since we control both the container
1249              * environment and the connection resource parameters, and the user
1250              * can use a different port if desired by setting control-port.
1251              */
1252             port->source = pcmk__itoa(DEFAULT_REMOTE_PORT);
1253         }
1254         port->target = strdup(port->source);
1255         bundle_data->ports = g_list_append(bundle_data->ports, port);
1256 
1257         buffer = calloc(1, max+1);
1258         for (childIter = bundle_data->child->children; childIter != NULL;
1259              childIter = childIter->next) {
1260 
1261             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1262 
1263             replica->child = childIter->data;
1264             replica->child->exclusive_discover = TRUE;
1265             replica->offset = lpc++;
1266 
1267             // Ensure the child's notify gets set based on the underlying primitive's value
1268             if (pcmk_is_set(replica->child->flags, pe_rsc_notify)) {
1269                 pe__set_resource_flags(bundle_data->child, pe_rsc_notify);
1270             }
1271 
1272             offset += allocate_ip(bundle_data, replica, buffer+offset,
1273                                   max-offset);
1274             bundle_data->replicas = g_list_append(bundle_data->replicas,
1275                                                   replica);
1276             bundle_data->attribute_target = g_hash_table_lookup(replica->child->meta,
1277                                                                 XML_RSC_ATTR_TARGET);
1278         }
1279         bundle_data->container_host_options = buffer;
1280         if (bundle_data->attribute_target) {
1281             g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET),
1282                                  strdup(bundle_data->attribute_target));
1283             g_hash_table_replace(bundle_data->child->meta,
1284                                  strdup(XML_RSC_ATTR_TARGET),
1285                                  strdup(bundle_data->attribute_target));
1286         }
1287 
1288     } else {
1289         // Just a naked container, no pacemaker-remote
1290         int offset = 0, max = 1024;
1291         char *buffer = calloc(1, max+1);
1292 
1293         for (int lpc = 0; lpc < bundle_data->nreplicas; lpc++) {
1294             pe__bundle_replica_t *replica = calloc(1, sizeof(pe__bundle_replica_t));
1295 
1296             replica->offset = lpc;
1297             offset += allocate_ip(bundle_data, replica, buffer+offset,
1298                                   max-offset);
1299             bundle_data->replicas = g_list_append(bundle_data->replicas,
1300                                                   replica);
1301         }
1302         bundle_data->container_host_options = buffer;
1303     }
1304 
1305     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1306          gIter = gIter->next) {
1307         pe__bundle_replica_t *replica = gIter->data;
1308 
1309         if (!create_container(rsc, bundle_data, replica, data_set)) {
1310             pe_err("Failed unpacking resource %s", rsc->id);
1311             rsc->fns->free(rsc);
1312             return FALSE;
1313         }
1314     }
1315 
1316     if (bundle_data->child) {
1317         rsc->children = g_list_append(rsc->children, bundle_data->child);
1318     }
1319     return TRUE;
1320 }
1321 
1322 static int
1323 replica_resource_active(pe_resource_t *rsc, gboolean all)
     /*  */
1324 {
1325     if (rsc) {
1326         gboolean child_active = rsc->fns->active(rsc, all);
1327 
1328         if (child_active && !all) {
1329             return TRUE;
1330         } else if (!child_active && all) {
1331             return FALSE;
1332         }
1333     }
1334     return -1;
1335 }
1336 
1337 gboolean
1338 pe__bundle_active(pe_resource_t *rsc, gboolean all)
     /*  */
1339 {
1340     pe__bundle_variant_data_t *bundle_data = NULL;
1341     GList *iter = NULL;
1342 
1343     get_bundle_variant_data(bundle_data, rsc);
1344     for (iter = bundle_data->replicas; iter != NULL; iter = iter->next) {
1345         pe__bundle_replica_t *replica = iter->data;
1346         int rsc_active;
1347 
1348         rsc_active = replica_resource_active(replica->ip, all);
1349         if (rsc_active >= 0) {
1350             return (gboolean) rsc_active;
1351         }
1352 
1353         rsc_active = replica_resource_active(replica->child, all);
1354         if (rsc_active >= 0) {
1355             return (gboolean) rsc_active;
1356         }
1357 
1358         rsc_active = replica_resource_active(replica->container, all);
1359         if (rsc_active >= 0) {
1360             return (gboolean) rsc_active;
1361         }
1362 
1363         rsc_active = replica_resource_active(replica->remote, all);
1364         if (rsc_active >= 0) {
1365             return (gboolean) rsc_active;
1366         }
1367     }
1368 
1369     /* If "all" is TRUE, we've already checked that no resources were inactive,
1370      * so return TRUE; if "all" is FALSE, we didn't find any active resources,
1371      * so return FALSE.
1372      */
1373     return all;
1374 }
1375 
1376 /*!
1377  * \internal
1378  * \brief Find the bundle replica corresponding to a given node
1379  *
1380  * \param[in] bundle  Top-level bundle resource
1381  * \param[in] node    Node to search for
1382  *
1383  * \return Bundle replica if found, NULL otherwise
1384  */
1385 pe_resource_t *
1386 pe__find_bundle_replica(const pe_resource_t *bundle, const pe_node_t *node)
     /*  */
1387 {
1388     pe__bundle_variant_data_t *bundle_data = NULL;
1389     CRM_ASSERT(bundle && node);
1390 
1391     get_bundle_variant_data(bundle_data, bundle);
1392     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1393          gIter = gIter->next) {
1394         pe__bundle_replica_t *replica = gIter->data;
1395 
1396         CRM_ASSERT(replica && replica->node);
1397         if (replica->node->details == node->details) {
1398             return replica->child;
1399         }
1400     }
1401     return NULL;
1402 }
1403 
1404 static void
1405 print_rsc_in_list(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1406                   void *print_data)
1407 {
1408     if (rsc != NULL) {
1409         if (options & pe_print_html) {
1410             status_print("<li>");
1411         }
1412         rsc->fns->print(rsc, pre_text, options, print_data);
1413         if (options & pe_print_html) {
1414             status_print("</li>\n");
1415         }
1416     }
1417 }
1418 
1419 static const char*
1420 container_agent_str(enum pe__container_agent t)
     /*  */
1421 {
1422     switch (t) {
1423         case PE__CONTAINER_AGENT_DOCKER: return PE__CONTAINER_AGENT_DOCKER_S;
1424         case PE__CONTAINER_AGENT_RKT:    return PE__CONTAINER_AGENT_RKT_S;
1425         case PE__CONTAINER_AGENT_PODMAN: return PE__CONTAINER_AGENT_PODMAN_S;
1426         default: // PE__CONTAINER_AGENT_UNKNOWN
1427             break;
1428     }
1429     return PE__CONTAINER_AGENT_UNKNOWN_S;
1430 }
1431 
1432 static void
1433 bundle_print_xml(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1434                  void *print_data)
1435 {
1436     pe__bundle_variant_data_t *bundle_data = NULL;
1437     char *child_text = NULL;
1438     CRM_CHECK(rsc != NULL, return);
1439 
1440     if (pre_text == NULL) {
1441         pre_text = "";
1442     }
1443     child_text = crm_strdup_printf("%s        ", pre_text);
1444 
1445     get_bundle_variant_data(bundle_data, rsc);
1446 
1447     status_print("%s<bundle ", pre_text);
1448     status_print("id=\"%s\" ", rsc->id);
1449     status_print("type=\"%s\" ", container_agent_str(bundle_data->agent_type));
1450     status_print("image=\"%s\" ", bundle_data->image);
1451     status_print("unique=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_unique));
1452     status_print("managed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_managed));
1453     status_print("failed=\"%s\" ", pe__rsc_bool_str(rsc, pe_rsc_failed));
1454     status_print(">\n");
1455 
1456     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1457          gIter = gIter->next) {
1458         pe__bundle_replica_t *replica = gIter->data;
1459 
1460         CRM_ASSERT(replica);
1461         status_print("%s    <replica id=\"%d\">\n", pre_text, replica->offset);
1462         print_rsc_in_list(replica->ip, child_text, options, print_data);
1463         print_rsc_in_list(replica->child, child_text, options, print_data);
1464         print_rsc_in_list(replica->container, child_text, options, print_data);
1465         print_rsc_in_list(replica->remote, child_text, options, print_data);
1466         status_print("%s    </replica>\n", pre_text);
1467     }
1468     status_print("%s</bundle>\n", pre_text);
1469     free(child_text);
1470 }
1471 
1472 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1473 int
1474 pe__bundle_xml(pcmk__output_t *out, va_list args)
1475 {
1476     unsigned int options = va_arg(args, unsigned int);
1477     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1478     GList *only_node = va_arg(args, GList *);
1479     GList *only_rsc = va_arg(args, GList *);
1480 
1481     pe__bundle_variant_data_t *bundle_data = NULL;
1482     int rc = pcmk_rc_no_output;
1483     gboolean printed_header = FALSE;
1484     gboolean print_everything = TRUE;
1485 
1486     CRM_ASSERT(rsc != NULL);
1487 
1488     get_bundle_variant_data(bundle_data, rsc);
1489 
1490     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1491         return rc;
1492     }
1493 
1494     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1495 
1496     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1497          gIter = gIter->next) {
1498         pe__bundle_replica_t *replica = gIter->data;
1499         char *id = pcmk__itoa(replica->offset);
1500         gboolean print_ip, print_child, print_ctnr, print_remote;
1501 
1502         CRM_ASSERT(replica);
1503 
1504         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1505             continue;
1506         }
1507 
1508         print_ip = replica->ip != NULL &&
1509                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1510         print_child = replica->child != NULL &&
1511                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1512         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1513         print_remote = replica->remote != NULL &&
1514                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1515 
1516         if (!print_everything && !print_ip && !print_child && !print_ctnr && !print_remote) {
1517             continue;
1518         }
1519 
1520         if (!printed_header) {
1521             printed_header = TRUE;
1522 
1523             rc = pe__name_and_nvpairs_xml(out, true, "bundle", 6,
1524                      "id", rsc->id,
1525                      "type", container_agent_str(bundle_data->agent_type),
1526                      "image", bundle_data->image,
1527                      "unique", pe__rsc_bool_str(rsc, pe_rsc_unique),
1528                      "managed", pe__rsc_bool_str(rsc, pe_rsc_managed),
1529                      "failed", pe__rsc_bool_str(rsc, pe_rsc_failed));
1530             CRM_ASSERT(rc == pcmk_rc_ok);
1531         }
1532 
1533         rc = pe__name_and_nvpairs_xml(out, true, "replica", 1, "id", id);
1534         free(id);
1535         CRM_ASSERT(rc == pcmk_rc_ok);
1536 
1537         if (print_ip) {
1538             out->message(out, crm_map_element_name(replica->ip->xml), options,
1539                          replica->ip, only_node, only_rsc);
1540         }
1541 
1542         if (print_child) {
1543             out->message(out, crm_map_element_name(replica->child->xml), options,
1544                          replica->child, only_node, only_rsc);
1545         }
1546 
1547         if (print_ctnr) {
1548             out->message(out, crm_map_element_name(replica->container->xml), options,
1549                          replica->container, only_node, only_rsc);
1550         }
1551 
1552         if (print_remote) {
1553             out->message(out, crm_map_element_name(replica->remote->xml), options,
1554                          replica->remote, only_node, only_rsc);
1555         }
1556 
1557         pcmk__output_xml_pop_parent(out); // replica
1558     }
1559 
1560     if (printed_header) {
1561         pcmk__output_xml_pop_parent(out); // bundle
1562     }
1563 
1564     return rc;
1565 }
1566 
1567 static void
1568 pe__bundle_replica_output_html(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1569                                pe_node_t *node, long options)
1570 {
1571     pe_resource_t *rsc = replica->child;
1572 
1573     int offset = 0;
1574     char buffer[LINE_MAX];
1575 
1576     if(rsc == NULL) {
1577         rsc = replica->container;
1578     }
1579 
1580     if (replica->remote) {
1581         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1582                            rsc_printable_id(replica->remote));
1583     } else {
1584         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1585                            rsc_printable_id(replica->container));
1586     }
1587     if (replica->ipaddr) {
1588         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1589                            replica->ipaddr);
1590     }
1591 
1592     pe__common_output_html(out, rsc, buffer, node, options);
1593 }
1594 
1595 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1596 int
1597 pe__bundle_html(pcmk__output_t *out, va_list args)
1598 {
1599     unsigned int options = va_arg(args, unsigned int);
1600     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1601     GList *only_node = va_arg(args, GList *);
1602     GList *only_rsc = va_arg(args, GList *);
1603 
1604     pe__bundle_variant_data_t *bundle_data = NULL;
1605     char buffer[LINE_MAX];
1606     int rc = pcmk_rc_no_output;
1607     gboolean print_everything = TRUE;
1608 
1609     CRM_ASSERT(rsc != NULL);
1610 
1611     get_bundle_variant_data(bundle_data, rsc);
1612 
1613     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1614         return rc;
1615     }
1616 
1617     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1618 
1619     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1620          gIter = gIter->next) {
1621         pe__bundle_replica_t *replica = gIter->data;
1622         gboolean print_ip, print_child, print_ctnr, print_remote;
1623 
1624         CRM_ASSERT(replica);
1625 
1626         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1627             continue;
1628         }
1629 
1630         print_ip = replica->ip != NULL &&
1631                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1632         print_child = replica->child != NULL &&
1633                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1634         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1635         print_remote = replica->remote != NULL &&
1636                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1637 
1638         if (pcmk_is_set(options, pe_print_implicit) ||
1639             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1640             /* The text output messages used below require pe_print_implicit to
1641              * be set to do anything.
1642              */
1643             unsigned int new_options = options;
1644 
1645             if (!pcmk_is_set(options, pe_print_implicit)) {
1646                 new_options |= pe_print_implicit;
1647             }
1648 
1649             if (rc == pcmk_rc_no_output) {
1650                 pcmk__output_create_xml_node(out, "br", NULL);
1651             }
1652 
1653             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1654                                      (bundle_data->nreplicas > 1)? " set" : "",
1655                                      rsc->id, bundle_data->image,
1656                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1657                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1658 
1659             pcmk__output_xml_create_parent(out, "li", NULL);
1660 
1661             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1662                 snprintf(buffer, LINE_MAX, " Replica[%d]", replica->offset);
1663                 xmlNodeSetContent(pcmk__output_xml_peek_parent(out), (pcmkXmlStr) buffer);
1664             }
1665             pcmk__output_create_xml_node(out, "br", NULL);
1666             out->begin_list(out, NULL, NULL, NULL);
1667 
1668             if (print_ip) {
1669                 out->message(out, crm_map_element_name(replica->ip->xml),
1670                              new_options, replica->ip, only_node, only_rsc);
1671             }
1672 
1673             if (print_child) {
1674                 out->message(out, crm_map_element_name(replica->child->xml),
1675                              new_options, replica->child, only_node, only_rsc);
1676             }
1677 
1678             if (print_ctnr) {
1679                 out->message(out, crm_map_element_name(replica->container->xml),
1680                              new_options, replica->container, only_node, only_rsc);
1681             }
1682 
1683             if (print_remote) {
1684                 out->message(out, crm_map_element_name(replica->remote->xml),
1685                              new_options, replica->remote, only_node, only_rsc);
1686             }
1687 
1688             out->end_list(out);
1689         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1690             continue;
1691         } else {
1692             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1693                                      (bundle_data->nreplicas > 1)? " set" : "",
1694                                      rsc->id, bundle_data->image,
1695                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1696                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1697 
1698             pe__bundle_replica_output_html(out, replica, pe__current_node(replica->container),
1699                                            options);
1700         }
1701 
1702         pcmk__output_xml_pop_parent(out);
1703     }
1704 
1705     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1706     return rc;
1707 }
1708 
1709 static void
1710 pe__bundle_replica_output_text(pcmk__output_t *out, pe__bundle_replica_t *replica,
     /*  */
1711                                pe_node_t *node, long options)
1712 {
1713     pe_resource_t *rsc = replica->child;
1714 
1715     int offset = 0;
1716     char buffer[LINE_MAX];
1717 
1718     if(rsc == NULL) {
1719         rsc = replica->container;
1720     }
1721 
1722     if (replica->remote) {
1723         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1724                            rsc_printable_id(replica->remote));
1725     } else {
1726         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1727                            rsc_printable_id(replica->container));
1728     }
1729     if (replica->ipaddr) {
1730         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1731                            replica->ipaddr);
1732     }
1733 
1734     pe__common_output_text(out, rsc, buffer, node, options);
1735 }
1736 
1737 PCMK__OUTPUT_ARGS("bundle", "unsigned int", "pe_resource_t *", "GList *", "GList *")
     /*  */
1738 int
1739 pe__bundle_text(pcmk__output_t *out, va_list args)
1740 {
1741     unsigned int options = va_arg(args, unsigned int);
1742     pe_resource_t *rsc = va_arg(args, pe_resource_t *);
1743     GList *only_node = va_arg(args, GList *);
1744     GList *only_rsc = va_arg(args, GList *);
1745 
1746     pe__bundle_variant_data_t *bundle_data = NULL;
1747     int rc = pcmk_rc_no_output;
1748     gboolean print_everything = TRUE;
1749 
1750     get_bundle_variant_data(bundle_data, rsc);
1751 
1752     CRM_ASSERT(rsc != NULL);
1753 
1754     if (rsc->fns->is_filtered(rsc, only_rsc, TRUE)) {
1755         return rc;
1756     }
1757 
1758     print_everything = pcmk__str_in_list(only_rsc, rsc->id);
1759 
1760     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1761          gIter = gIter->next) {
1762         pe__bundle_replica_t *replica = gIter->data;
1763         gboolean print_ip, print_child, print_ctnr, print_remote;
1764 
1765         CRM_ASSERT(replica);
1766 
1767         if (pcmk__rsc_filtered_by_node(replica->container, only_node)) {
1768             continue;
1769         }
1770 
1771         print_ip = replica->ip != NULL &&
1772                    !replica->ip->fns->is_filtered(replica->ip, only_rsc, print_everything);
1773         print_child = replica->child != NULL &&
1774                       !replica->child->fns->is_filtered(replica->child, only_rsc, print_everything);
1775         print_ctnr = !replica->container->fns->is_filtered(replica->container, only_rsc, print_everything);
1776         print_remote = replica->remote != NULL &&
1777                        !replica->remote->fns->is_filtered(replica->remote, only_rsc, print_everything);
1778 
1779         if (pcmk_is_set(options, pe_print_implicit) ||
1780             (print_everything == FALSE && (print_ip || print_child || print_ctnr || print_remote))) {
1781             /* The text output messages used below require pe_print_implicit to
1782              * be set to do anything.
1783              */
1784             unsigned int new_options = options;
1785 
1786             if (!pcmk_is_set(options, pe_print_implicit)) {
1787                 new_options |= pe_print_implicit;
1788             }
1789             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1790                                      (bundle_data->nreplicas > 1)? " set" : "",
1791                                      rsc->id, bundle_data->image,
1792                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1793                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1794 
1795             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1796                 out->list_item(out, NULL, "Replica[%d]", replica->offset);
1797             }
1798 
1799             out->begin_list(out, NULL, NULL, NULL);
1800 
1801             if (print_ip) {
1802                 out->message(out, crm_map_element_name(replica->ip->xml),
1803                              new_options, replica->ip, only_node, only_rsc);
1804             }
1805 
1806             if (print_child) {
1807                 out->message(out, crm_map_element_name(replica->child->xml),
1808                              new_options, replica->child, only_node, only_rsc);
1809             }
1810 
1811             if (print_ctnr) {
1812                 out->message(out, crm_map_element_name(replica->container->xml),
1813                              new_options, replica->container, only_node, only_rsc);
1814             }
1815 
1816             if (print_remote) {
1817                 out->message(out, crm_map_element_name(replica->remote->xml),
1818                              new_options, replica->remote, only_node, only_rsc);
1819             }
1820 
1821             out->end_list(out);
1822         } else if (print_everything == FALSE && !(print_ip || print_child || print_ctnr || print_remote)) {
1823             continue;
1824         } else {
1825             PCMK__OUTPUT_LIST_HEADER(out, FALSE, rc, "Container bundle%s: %s [%s]%s%s",
1826                                      (bundle_data->nreplicas > 1)? " set" : "",
1827                                      rsc->id, bundle_data->image,
1828                                      pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1829                                      pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1830 
1831             pe__bundle_replica_output_text(out, replica, pe__current_node(replica->container),
1832                                            options);
1833         }
1834     }
1835 
1836     PCMK__OUTPUT_LIST_FOOTER(out, rc);
1837     return rc;
1838 }
1839 
1840 static void
1841 print_bundle_replica(pe__bundle_replica_t *replica, const char *pre_text,
     /*  */
1842                      long options, void *print_data)
1843 {
1844     pe_node_t *node = NULL;
1845     pe_resource_t *rsc = replica->child;
1846 
1847     int offset = 0;
1848     char buffer[LINE_MAX];
1849 
1850     if(rsc == NULL) {
1851         rsc = replica->container;
1852     }
1853 
1854     if (replica->remote) {
1855         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1856                            rsc_printable_id(replica->remote));
1857     } else {
1858         offset += snprintf(buffer + offset, LINE_MAX - offset, "%s",
1859                            rsc_printable_id(replica->container));
1860     }
1861     if (replica->ipaddr) {
1862         offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)",
1863                            replica->ipaddr);
1864     }
1865 
1866     node = pe__current_node(replica->container);
1867     common_print(rsc, pre_text, buffer, node, options, print_data);
1868 }
1869 
1870 void
1871 pe__print_bundle(pe_resource_t *rsc, const char *pre_text, long options,
     /*  */
1872                  void *print_data)
1873 {
1874     pe__bundle_variant_data_t *bundle_data = NULL;
1875     char *child_text = NULL;
1876     CRM_CHECK(rsc != NULL, return);
1877 
1878     if (options & pe_print_xml) {
1879         bundle_print_xml(rsc, pre_text, options, print_data);
1880         return;
1881     }
1882 
1883     get_bundle_variant_data(bundle_data, rsc);
1884 
1885     if (pre_text == NULL) {
1886         pre_text = " ";
1887     }
1888 
1889     status_print("%sContainer bundle%s: %s [%s]%s%s\n",
1890                  pre_text, ((bundle_data->nreplicas > 1)? " set" : ""),
1891                  rsc->id, bundle_data->image,
1892                  pcmk_is_set(rsc->flags, pe_rsc_unique) ? " (unique)" : "",
1893                  pcmk_is_set(rsc->flags, pe_rsc_managed) ? "" : " (unmanaged)");
1894     if (options & pe_print_html) {
1895         status_print("<br />\n<ul>\n");
1896     }
1897 
1898 
1899     for (GList *gIter = bundle_data->replicas; gIter != NULL;
1900          gIter = gIter->next) {
1901         pe__bundle_replica_t *replica = gIter->data;
1902 
1903         CRM_ASSERT(replica);
1904         if (options & pe_print_html) {
1905             status_print("<li>");
1906         }
1907 
1908         if (pcmk_is_set(options, pe_print_implicit)) {
1909             child_text = crm_strdup_printf("     %s", pre_text);
1910             if (pcmk__list_of_multiple(bundle_data->replicas)) {
1911                 status_print("  %sReplica[%d]\n", pre_text, replica->offset);
1912             }
1913             if (options & pe_print_html) {
1914                 status_print("<br />\n<ul>\n");
1915             }
1916             print_rsc_in_list(replica->ip, child_text, options, print_data);
1917             print_rsc_in_list(replica->container, child_text, options, print_data);
1918             print_rsc_in_list(replica->remote, child_text, options, print_data);
1919             print_rsc_in_list(replica->child, child_text, options, print_data);
1920             if (options & pe_print_html) {
1921                 status_print("</ul>\n");
1922             }
1923         } else {
1924             child_text = crm_strdup_printf("%s  ", pre_text);
1925             print_bundle_replica(replica, child_text, options, print_data);
1926         }
1927         free(child_text);
1928 
1929         if (options & pe_print_html) {
1930             status_print("</li>\n");
1931         }
1932     }
1933     if (options & pe_print_html) {
1934         status_print("</ul>\n");
1935     }
1936 }
1937 
1938 static void
1939 free_bundle_replica(pe__bundle_replica_t *replica)
     /*  */
1940 {
1941     if (replica == NULL) {
1942         return;
1943     }
1944 
1945     if (replica->node) {
1946         free(replica->node);
1947         replica->node = NULL;
1948     }
1949 
1950     if (replica->ip) {
1951         free_xml(replica->ip->xml);
1952         replica->ip->xml = NULL;
1953         replica->ip->fns->free(replica->ip);
1954         replica->ip = NULL;
1955     }
1956     if (replica->container) {
1957         free_xml(replica->container->xml);
1958         replica->container->xml = NULL;
1959         replica->container->fns->free(replica->container);
1960         replica->container = NULL;
1961     }
1962     if (replica->remote) {
1963         free_xml(replica->remote->xml);
1964         replica->remote->xml = NULL;
1965         replica->remote->fns->free(replica->remote);
1966         replica->remote = NULL;
1967     }
1968     free(replica->ipaddr);
1969     free(replica);
1970 }
1971 
1972 void
1973 pe__free_bundle(pe_resource_t *rsc)
     /*  */
1974 {
1975     pe__bundle_variant_data_t *bundle_data = NULL;
1976     CRM_CHECK(rsc != NULL, return);
1977 
1978     get_bundle_variant_data(bundle_data, rsc);
1979     pe_rsc_trace(rsc, "Freeing %s", rsc->id);
1980 
1981     free(bundle_data->prefix);
1982     free(bundle_data->image);
1983     free(bundle_data->control_port);
1984     free(bundle_data->host_network);
1985     free(bundle_data->host_netmask);
1986     free(bundle_data->ip_range_start);
1987     free(bundle_data->container_network);
1988     free(bundle_data->launcher_options);
1989     free(bundle_data->container_command);
1990     free(bundle_data->container_host_options);
1991 
1992     g_list_free_full(bundle_data->replicas,
1993                      (GDestroyNotify) free_bundle_replica);
1994     g_list_free_full(bundle_data->mounts, (GDestroyNotify)mount_free);
1995     g_list_free_full(bundle_data->ports, (GDestroyNotify)port_free);
1996     g_list_free(rsc->children);
1997 
1998     if(bundle_data->child) {
1999         free_xml(bundle_data->child->xml);
2000         bundle_data->child->xml = NULL;
2001         bundle_data->child->fns->free(bundle_data->child);
2002     }
2003     common_free(rsc);
2004 }
2005 
2006 enum rsc_role_e
2007 pe__bundle_resource_state(const pe_resource_t *rsc, gboolean current)
     /*  */
2008 {
2009     enum rsc_role_e container_role = RSC_ROLE_UNKNOWN;
2010     return container_role;
2011 }
2012 
2013 /*!
2014  * \brief Get the number of configured replicas in a bundle
2015  *
2016  * \param[in] rsc  Bundle resource
2017  *
2018  * \return Number of configured replicas, or 0 on error
2019  */
2020 int
2021 pe_bundle_replicas(const pe_resource_t *rsc)
     /*  */
2022 {
2023     if ((rsc == NULL) || (rsc->variant != pe_container)) {
2024         return 0;
2025     } else {
2026         pe__bundle_variant_data_t *bundle_data = NULL;
2027 
2028         get_bundle_variant_data(bundle_data, rsc);
2029         return bundle_data->nreplicas;
2030     }
2031 }
2032 
2033 void
2034 pe__count_bundle(pe_resource_t *rsc)
     /*  */
2035 {
2036     pe__bundle_variant_data_t *bundle_data = NULL;
2037 
2038     get_bundle_variant_data(bundle_data, rsc);
2039     for (GList *item = bundle_data->replicas; item != NULL; item = item->next) {
2040         pe__bundle_replica_t *replica = item->data;
2041 
2042         if (replica->ip) {
2043             replica->ip->fns->count(replica->ip);
2044         }
2045         if (replica->child) {
2046             replica->child->fns->count(replica->child);
2047         }
2048         if (replica->container) {
2049             replica->container->fns->count(replica->container);
2050         }
2051         if (replica->remote) {
2052             replica->remote->fns->count(replica->remote);
2053         }
2054     }
2055 }
2056 
2057 gboolean
2058 pe__bundle_is_filtered(pe_resource_t *rsc, GList *only_rsc, gboolean check_parent)
     /*  */
2059 {
2060     gboolean passes = FALSE;
2061     pe__bundle_variant_data_t *bundle_data = NULL;
2062 
2063     if (pcmk__str_in_list(only_rsc, rsc_printable_id(rsc))) {
2064         passes = TRUE;
2065     } else {
2066         get_bundle_variant_data(bundle_data, rsc);
2067 
2068         for (GList *gIter = bundle_data->replicas; gIter != NULL; gIter = gIter->next) {
2069             pe__bundle_replica_t *replica = gIter->data;
2070 
2071             if (replica->ip != NULL && !replica->ip->fns->is_filtered(replica->ip, only_rsc, FALSE)) {
2072                 passes = TRUE;
2073                 break;
2074             } else if (replica->child != NULL && !replica->child->fns->is_filtered(replica->child, only_rsc, FALSE)) {
2075                 passes = TRUE;
2076                 break;
2077             } else if (!replica->container->fns->is_filtered(replica->container, only_rsc, FALSE)) {
2078                 passes = TRUE;
2079                 break;
2080             } else if (replica->remote != NULL && !replica->remote->fns->is_filtered(replica->remote, only_rsc, FALSE)) {
2081                 passes = TRUE;
2082                 break;
2083             }
2084         }
2085     }
2086 
2087     return !passes;
2088 }
/* */
root/lib/pengine/bundle.c

DEFINITIONS
