root/pengine/container.c

/* */

DEFINITIONS

1. is_child_container_node
2. get_container_list
3. get_containers_or_children
4. migration_threshold_reached
5. container_color
6. container_create_actions
7. container_internal_constraints
8. find_compatible_tuple_by_node
9. find_compatible_tuple
10. container_rsc_colocation_lh
11. copies_per_node
12. container_rsc_colocation_rh
13. container_action_flags
14. find_compatible_child_by_node
15. tuple_for_docker
16. container_update_interleave_actions
17. can_interleave_actions
18. container_update_actions
19. container_rsc_location
20. container_expand
21. container_create_probe
22. container_append_meta
23. container_merge_weights
24. container_LogActions

   1 /* 
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  * 
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  * 
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  * 
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <crm/msg_xml.h>
  22 #include <allocate.h>
  23 #include <notif.h>
  24 #include <utils.h>
  25 
  26 #define VARIANT_CONTAINER 1
  27 #include <lib/pengine/variant.h>
  28 
  29 static bool
  30 is_child_container_node(container_variant_data_t *data, pe_node_t *node)
     /*  */
  31 {
  32     for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
  33         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
  34         if(node->details == tuple->node->details) {
  35             return TRUE;
  36         }
  37     }
  38     return FALSE;
  39 }
  40 
  41 gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
  42 void distribute_children(resource_t *rsc, GListPtr children, GListPtr nodes,
  43                          int max, int per_host_max, pe_working_set_t * data_set);
  44 
  45 static GListPtr get_container_list(resource_t *rsc) 
     /*  */
  46 {
  47     GListPtr containers = NULL;
  48     container_variant_data_t *data = NULL;
  49 
  50     if(rsc->variant == pe_container) {
  51         get_container_variant_data(data, rsc);
  52         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
  53             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
  54             containers = g_list_append(containers, tuple->docker);
  55         }
  56     }
  57     return containers;
  58 }
  59 
  60 static GListPtr get_containers_or_children(resource_t *rsc) 
     /*  */
  61 {
  62     GListPtr containers = NULL;
  63     container_variant_data_t *data = NULL;
  64 
  65     if(rsc->variant == pe_container) {
  66         get_container_variant_data(data, rsc);
  67         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
  68             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
  69             containers = g_list_append(containers, tuple->docker);
  70         }
  71         return containers;
  72     } else {
  73         return rsc->children;
  74     }
  75 }
  76 
  77 static bool
  78 migration_threshold_reached(resource_t *rsc, node_t *node,
     /*  */
  79                             pe_working_set_t *data_set)
  80 {
  81     int fail_count, countdown;
  82 
  83     /* Migration threshold of 0 means never force away */
  84     if (rsc->migration_threshold == 0) {
  85         return FALSE;
  86     }
  87 
  88     // If we're ignoring failures, also ignore the migration threshold
  89     if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
  90         return FALSE;
  91     }
  92 
  93     /* If there are no failures, there's no need to force away */
  94     fail_count = pe_get_failcount(node, rsc, NULL,
  95                                   pe_fc_effective|pe_fc_fillers, NULL,
  96                                   data_set);
  97     if (fail_count <= 0) {
  98         return FALSE;
  99     }
 100 
 101     /* How many more times recovery will be tried on this node */
 102     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 103 
 104     if (countdown == 0) {
 105         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
 106                  rsc->id, node->details->uname, fail_count,
 107                  rsc->migration_threshold);
 108         return TRUE;
 109     }
 110 
 111     crm_info("%s can fail %d more times on %s before being forced off",
 112              rsc->id, countdown, node->details->uname);
 113     return FALSE;
 114 }
 115 
 116 node_t *
 117 container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
     /*  */
 118 {
 119     GListPtr containers = NULL;
 120     GListPtr nodes = NULL;
 121     container_variant_data_t *container_data = NULL;
 122 
 123     CRM_CHECK(rsc != NULL, return NULL);
 124 
 125     get_container_variant_data(container_data, rsc);
 126 
 127     set_bit(rsc->flags, pe_rsc_allocating);
 128     containers = get_container_list(rsc);
 129 
 130     dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes);
 131 
 132     nodes = g_hash_table_get_values(rsc->allowed_nodes);
 133     nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);
 134     containers = g_list_sort_with_data(containers, sort_clone_instance, data_set);
 135     distribute_children(rsc, containers, nodes,
 136                         container_data->replicas, container_data->replicas_per_host, data_set);
 137     g_list_free(nodes);
 138     g_list_free(containers);
 139 
 140     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 141         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 142         pe_node_t *docker_host = tuple->docker->allocated_to;
 143 
 144         CRM_ASSERT(tuple);
 145         if(tuple->ip) {
 146             tuple->ip->cmds->allocate(tuple->ip, prefer, data_set);
 147         }
 148 
 149         if(tuple->remote && is_remote_node(docker_host)) {
 150             /* We need 'nested' connection resources to be on the same
 151              * host because pacemaker-remoted only supports a single
 152              * active connection
 153              */
 154             rsc_colocation_new("child-remote-with-docker-remote", NULL,
 155                                INFINITY, tuple->remote, docker_host->details->remote_rsc, NULL, NULL, data_set);
 156         }
 157 
 158         if(tuple->remote) {
 159             tuple->remote->cmds->allocate(tuple->remote, prefer, data_set);
 160         }
 161 
 162         // Explicitly allocate tuple->child before the container->child
 163         if(tuple->child) {
 164             pe_node_t *node = NULL;
 165             GHashTableIter iter;
 166             g_hash_table_iter_init(&iter, tuple->child->allowed_nodes);
 167             while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
 168                 if(node->details != tuple->node->details) {
 169                     node->weight = -INFINITY;
 170                 } else if(migration_threshold_reached(tuple->child, node, data_set) == FALSE) {
 171                     node->weight = INFINITY;
 172                 }
 173             }
 174 
 175             set_bit(tuple->child->parent->flags, pe_rsc_allocating);
 176             tuple->child->cmds->allocate(tuple->child, tuple->node, data_set);
 177             clear_bit(tuple->child->parent->flags, pe_rsc_allocating);
 178         }
 179     }
 180 
 181     if(container_data->child) {
 182         pe_node_t *node = NULL;
 183         GHashTableIter iter;
 184         g_hash_table_iter_init(&iter, container_data->child->allowed_nodes);
 185         while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) {
 186             if(is_child_container_node(container_data, node)) {
 187                 node->weight = 0;
 188             } else {
 189                 node->weight = -INFINITY;
 190             }
 191         }
 192         container_data->child->cmds->allocate(container_data->child, prefer, data_set);
 193     }
 194 
 195     clear_bit(rsc->flags, pe_rsc_allocating);
 196     clear_bit(rsc->flags, pe_rsc_provisional);
 197     return NULL;
 198 }
 199 
 200 
 201 void
 202 container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
 203 {
 204     pe_action_t *action = NULL;
 205     GListPtr containers = NULL;
 206     container_variant_data_t *container_data = NULL;
 207 
 208     CRM_CHECK(rsc != NULL, return);
 209 
 210     containers = get_container_list(rsc);
 211     get_container_variant_data(container_data, rsc);
 212     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 213         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 214 
 215         CRM_ASSERT(tuple);
 216         if(tuple->ip) {
 217             tuple->ip->cmds->create_actions(tuple->ip, data_set);
 218         }
 219         if(tuple->docker) {
 220             tuple->docker->cmds->create_actions(tuple->docker, data_set);
 221         }
 222         if(tuple->remote) {
 223             tuple->remote->cmds->create_actions(tuple->remote, data_set);
 224         }
 225     }
 226 
 227     clone_create_pseudo_actions(rsc, containers, NULL, NULL,  data_set);
 228 
 229     if(container_data->child) {
 230         container_data->child->cmds->create_actions(container_data->child, data_set);
 231 
 232         if(container_data->child->variant == pe_master) {
 233             /* promote */
 234             action = create_pseudo_resource_op(rsc, RSC_PROMOTE, TRUE, TRUE, data_set);
 235             action = create_pseudo_resource_op(rsc, RSC_PROMOTED, TRUE, TRUE, data_set);
 236             action->priority = INFINITY;
 237 
 238             /* demote */
 239             action = create_pseudo_resource_op(rsc, RSC_DEMOTE, TRUE, TRUE, data_set);
 240             action = create_pseudo_resource_op(rsc, RSC_DEMOTED, TRUE, TRUE, data_set);
 241             action->priority = INFINITY;
 242         }
 243     }
 244 
 245     g_list_free(containers);
 246 }
 247 
 248 void
 249 container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
 250 {
 251     container_variant_data_t *container_data = NULL;
 252 
 253     CRM_CHECK(rsc != NULL, return);
 254 
 255     get_container_variant_data(container_data, rsc);
 256 
 257     if(container_data->child) {
 258         new_rsc_order(rsc, RSC_START, container_data->child, RSC_START, pe_order_implies_first_printed, data_set);
 259         new_rsc_order(rsc, RSC_STOP, container_data->child, RSC_STOP, pe_order_implies_first_printed, data_set);
 260 
 261         if(container_data->child->children) {
 262             new_rsc_order(container_data->child, RSC_STARTED, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
 263             new_rsc_order(container_data->child, RSC_STOPPED, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
 264         } else {
 265             new_rsc_order(container_data->child, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
 266             new_rsc_order(container_data->child, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
 267         }
 268     }
 269 
 270     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 271         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 272 
 273         CRM_ASSERT(tuple);
 274         CRM_ASSERT(tuple->docker);
 275 
 276         tuple->docker->cmds->internal_constraints(tuple->docker, data_set);
 277 
 278         order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed);
 279 
 280         if(tuple->child) {
 281             order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
 282         }
 283         order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
 284         new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
 285         new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
 286 
 287         if(tuple->ip) {
 288             tuple->ip->cmds->internal_constraints(tuple->ip, data_set);
 289 
 290             // Start ip then docker
 291             new_rsc_order(tuple->ip, RSC_START, tuple->docker, RSC_START,
 292                           pe_order_runnable_left|pe_order_preserve, data_set);
 293             new_rsc_order(tuple->docker, RSC_STOP, tuple->ip, RSC_STOP,
 294                           pe_order_implies_first|pe_order_preserve, data_set);
 295 
 296             rsc_colocation_new("ip-with-docker", NULL, INFINITY, tuple->ip, tuple->docker, NULL, NULL, data_set);
 297         }
 298 
 299         if(tuple->remote) {
 300             /* This handles ordering and colocating remote relative to docker
 301              * (via "resource-with-container"). Since IP is also ordered and
 302              * colocated relative to docker, we don't need to do anything
 303              * explicit here with IP.
 304              */
 305             tuple->remote->cmds->internal_constraints(tuple->remote, data_set);
 306         }
 307 
 308         if(tuple->child) {
 309             CRM_ASSERT(tuple->remote);
 310 
 311             // Start of the remote then child is implicit in the PE's remote logic
 312         }
 313 
 314     }
 315 
 316     if(container_data->child) {
 317         container_data->child->cmds->internal_constraints(container_data->child, data_set);
 318         if(container_data->child->variant == pe_master) {
 319             master_promotion_constraints(rsc, data_set);
 320 
 321             /* child demoted before global demoted */
 322             new_rsc_order(container_data->child, RSC_DEMOTED, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set);
 323 
 324             /* global demote before child demote */
 325             new_rsc_order(rsc, RSC_DEMOTE, container_data->child, RSC_DEMOTE, pe_order_implies_first_printed, data_set);
 326 
 327             /* child promoted before global promoted */
 328             new_rsc_order(container_data->child, RSC_PROMOTED, rsc, RSC_PROMOTED, pe_order_implies_then_printed, data_set);
 329 
 330             /* global promote before child promote */
 331             new_rsc_order(rsc, RSC_PROMOTE, container_data->child, RSC_PROMOTE, pe_order_implies_first_printed, data_set);
 332         }
 333 
 334     } else {
 335 //    int type = pe_order_optional | pe_order_implies_then | pe_order_restart;
 336 //        custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
 337 //                            rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set);
 338     }
 339 }
 340 
 341 
 342 
 343 static resource_t *
 344 find_compatible_tuple_by_node(resource_t * rsc_lh, node_t * candidate, resource_t * rsc,
     /*  */
 345                               enum rsc_role_e filter, gboolean current)
 346 {
 347     container_variant_data_t *container_data = NULL;
 348 
 349     CRM_CHECK(candidate != NULL, return NULL);
 350     get_container_variant_data(container_data, rsc);
 351 
 352     crm_trace("Looking for compatible child from %s for %s on %s",
 353               rsc_lh->id, rsc->id, candidate->details->uname);
 354 
 355     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 356         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 357 
 358         if(is_child_compatible(tuple->docker, candidate, filter, current)) {
 359             crm_trace("Pairing %s with %s on %s",
 360                       rsc_lh->id, tuple->docker->id, candidate->details->uname);
 361             return tuple->docker;
 362         }
 363     }
 364 
 365     crm_trace("Can't pair %s with %s", rsc_lh->id, rsc->id);
 366     return NULL;
 367 }
 368 
 369 static resource_t *
 370 find_compatible_tuple(resource_t *rsc_lh, resource_t * rsc, enum rsc_role_e filter,
     /*  */
 371                       gboolean current)
 372 {
 373     GListPtr scratch = NULL;
 374     resource_t *pair = NULL;
 375     node_t *active_node_lh = NULL;
 376 
 377     active_node_lh = rsc_lh->fns->location(rsc_lh, NULL, current);
 378     if (active_node_lh) {
 379         return find_compatible_tuple_by_node(rsc_lh, active_node_lh, rsc, filter, current);
 380     }
 381 
 382     scratch = g_hash_table_get_values(rsc_lh->allowed_nodes);
 383     scratch = g_list_sort_with_data(scratch, sort_node_weight, NULL);
 384 
 385     for (GListPtr gIter = scratch; gIter != NULL; gIter = gIter->next) {
 386         node_t *node = (node_t *) gIter->data;
 387 
 388         pair = find_compatible_tuple_by_node(rsc_lh, node, rsc, filter, current);
 389         if (pair) {
 390             goto done;
 391         }
 392     }
 393 
 394     pe_rsc_debug(rsc, "Can't pair %s with %s", rsc_lh->id, rsc->id);
 395   done:
 396     g_list_free(scratch);
 397     return pair;
 398 }
 399 
 400 void
 401 container_rsc_colocation_lh(resource_t * rsc, resource_t * rsc_rh, rsc_colocation_t * constraint)
     /*  */
 402 {
 403     /* -- Never called --
 404      *
 405      * Instead we add the colocation constraints to the child and call from there
 406      */
 407     CRM_ASSERT(FALSE);
 408 }
 409 
 410 int copies_per_node(resource_t * rsc) 
     /*  */
 411 {
 412     /* Strictly speaking, there should be a 'copies_per_node' addition
 413      * to the resource function table and each case would be a
 414      * function.  However that would be serious overkill to return an
 415      * int.  In fact, it seems to me that both function tables
 416      * could/should be replaced by resources.{c,h} full of
 417      * rsc_{some_operation} functions containing a switch as below
 418      * which calls out to functions named {variant}_{some_operation}
 419      * as needed.
 420      */
 421     switch(rsc->variant) {
 422         case pe_unknown:
 423             return 0;
 424         case pe_native:
 425         case pe_group:
 426             return 1;
 427         case pe_clone:
 428         case pe_master:
 429             {
 430                 const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);
 431                 return crm_parse_int(max_clones_node, "1");
 432             }
 433         case pe_container:
 434             {
 435                 container_variant_data_t *data = NULL;
 436                 get_container_variant_data(data, rsc);
 437                 return data->replicas_per_host;
 438             }
 439     }
 440     return 0;
 441 }
 442 
 443 void
 444 container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocation_t * constraint)
     /*  */
 445 {
 446     GListPtr allocated_rhs = NULL;
 447     container_variant_data_t *container_data = NULL;
 448 
 449     CRM_CHECK(constraint != NULL, return);
 450     CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
 451     CRM_CHECK(rsc != NULL, pe_err("rsc was NULL for %s", constraint->id); return);
 452     CRM_ASSERT(rsc_lh->variant == pe_native);
 453 
 454     if (is_set(rsc->flags, pe_rsc_provisional)) {
 455         pe_rsc_trace(rsc, "%s is still provisional", rsc->id);
 456         return;
 457 
 458     } else if(constraint->rsc_lh->variant > pe_group) {
 459         resource_t *rh_child = find_compatible_tuple(rsc_lh, rsc, RSC_ROLE_UNKNOWN, FALSE);
 460 
 461         if (rh_child) {
 462             pe_rsc_debug(rsc, "Pairing %s with %s", rsc_lh->id, rh_child->id);
 463             rsc_lh->cmds->rsc_colocation_lh(rsc_lh, rh_child, constraint);
 464 
 465         } else if (constraint->score >= INFINITY) {
 466             crm_notice("Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
 467             assign_node(rsc_lh, NULL, TRUE);
 468 
 469         } else {
 470             pe_rsc_debug(rsc, "Cannot pair %s with instance of %s", rsc_lh->id, rsc->id);
 471         }
 472 
 473         return;
 474     }
 475 
 476     get_container_variant_data(container_data, rsc);
 477     pe_rsc_trace(rsc, "Processing constraint %s: %s -> %s %d",
 478                  constraint->id, rsc_lh->id, rsc->id, constraint->score);
 479 
 480     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 481         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 482 
 483         if (constraint->score < INFINITY) {
 484             tuple->docker->cmds->rsc_colocation_rh(rsc_lh, tuple->docker, constraint);
 485 
 486         } else {
 487             node_t *chosen = tuple->docker->fns->location(tuple->docker, NULL, FALSE);
 488 
 489             if (chosen != NULL && is_set_recursive(tuple->docker, pe_rsc_block, TRUE) == FALSE) {
 490                 pe_rsc_trace(rsc, "Allowing %s: %s %d", constraint->id, chosen->details->uname, chosen->weight);
 491                 allocated_rhs = g_list_prepend(allocated_rhs, chosen);
 492             }
 493         }
 494     }
 495 
 496     if (constraint->score >= INFINITY) {
 497         node_list_exclude(rsc_lh->allowed_nodes, allocated_rhs, FALSE);
 498     }
 499     g_list_free(allocated_rhs);
 500 }
 501 
 502 enum pe_action_flags
 503 container_action_flags(action_t * action, node_t * node)
     /*  */
 504 {
 505     GListPtr containers = NULL;
 506     enum pe_action_flags flags = 0;
 507     container_variant_data_t *data = NULL;
 508 
 509     get_container_variant_data(data, action->rsc);
 510     if(data->child) {
 511         enum action_tasks task = get_complex_task(data->child, action->task, TRUE);
 512         switch(task) {
 513             case no_action:
 514             case action_notify:
 515             case action_notified:
 516             case action_promote:
 517             case action_promoted:
 518             case action_demote:
 519             case action_demoted:
 520                 return summary_action_flags(action, data->child->children, node);
 521             default:
 522                 break;
 523         }
 524     }
 525 
 526     containers = get_container_list(action->rsc);
 527     flags = summary_action_flags(action, containers, node);
 528     g_list_free(containers);
 529     return flags;
 530 }
 531 
 532 resource_t *
 533 find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
     /*  */
 534                               enum rsc_role_e filter, gboolean current)
 535 {
 536     GListPtr gIter = NULL;
 537     GListPtr children = NULL;
 538 
 539     if (local_node == NULL) {
 540         crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
 541         return NULL;
 542     }
 543 
 544     crm_trace("Looking for compatible child from %s for %s on %s",
 545               local_child->id, rsc->id, local_node->details->uname);
 546 
 547     children = get_containers_or_children(rsc);
 548     for (gIter = children; gIter != NULL; gIter = gIter->next) {
 549         resource_t *child_rsc = (resource_t *) gIter->data;
 550 
 551         if(is_child_compatible(child_rsc, local_node, filter, current)) {
 552             crm_trace("Pairing %s with %s on %s",
 553                       local_child->id, child_rsc->id, local_node->details->uname);
 554             return child_rsc;
 555         }
 556     }
 557 
 558     crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
 559     if(children != rsc->children) {
 560         g_list_free(children);
 561     }
 562     return NULL;
 563 }
 564 
 565 static container_grouping_t *
 566 tuple_for_docker(resource_t *rsc, resource_t *docker, node_t *node)
     /*  */
 567 {
 568     if(rsc->variant == pe_container) {
 569         container_variant_data_t *data = NULL;
 570         get_container_variant_data(data, rsc);
 571         for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
 572             container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 573             if(tuple->child
 574                && docker == tuple->docker
 575                && node->details == tuple->node->details) {
 576                 return tuple;
 577             }
 578         }
 579     }
 580     return NULL;
 581 }
 582 
 583 static enum pe_graph_flags
 584 container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
     /*  */
 585                      enum pe_action_flags filter, enum pe_ordering type)
 586 {
 587     GListPtr gIter = NULL;
 588     GListPtr children = NULL;
 589     gboolean current = FALSE;
 590     enum pe_graph_flags changed = pe_graph_none;
 591 
 592     /* Fix this - lazy */
 593     if (crm_ends_with(first->uuid, "_stopped_0")
 594         || crm_ends_with(first->uuid, "_demoted_0")) {
 595         current = TRUE;
 596     }
 597 
 598     children = get_containers_or_children(then->rsc);
 599     for (gIter = children; gIter != NULL; gIter = gIter->next) {
 600         resource_t *then_child = (resource_t *) gIter->data;
 601         resource_t *first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current);
 602         if (first_child == NULL && current) {
 603             crm_trace("Ignore");
 604 
 605         } else if (first_child == NULL) {
 606             crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, then->uuid);
 607 
 608             /* Me no like this hack - but what else can we do?
 609              *
 610              * If there is no-one active or about to be active
 611              *   on the same node as then_child, then they must
 612              *   not be allowed to start
 613              */
 614             if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
 615                 pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
 616                 if(assign_node(then_child, NULL, TRUE)) {
 617                     changed |= pe_graph_updated_then;
 618                 }
 619             }
 620 
 621         } else {
 622             pe_action_t *first_action = NULL;
 623             pe_action_t *then_action = NULL;
 624 
 625             enum action_tasks task = clone_child_action(first);
 626             const char *first_task = task2text(task);
 627 
 628             container_grouping_t *first_tuple = tuple_for_docker(first->rsc, first_child, node);
 629             container_grouping_t *then_tuple = tuple_for_docker(then->rsc, then_child, node);
 630 
 631             if(strstr(first->task, "stop") && first_tuple && first_tuple->child) {
 632                 /* Except for 'stopped' we should be looking at the
 633                  * in-container resource, actions for the child will
 634                  * happen later and are therefor more likely to align
 635                  * with the user's intent.
 636                  */
 637                 first_action = find_first_action(first_tuple->child->actions, NULL, task2text(task), node);
 638             } else {
 639                 first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
 640             }
 641 
 642             if(strstr(then->task, "mote") && then_tuple && then_tuple->child) {
 643                 /* Promote/demote actions will never be found for the
 644                  * docker resource, look in the child instead
 645                  *
 646                  * Alternatively treat:
 647                  *  'XXXX then promote YYYY' as 'XXXX then start container for YYYY', and
 648                  *  'demote XXXX then stop YYYY' as 'stop container for XXXX then stop YYYY'
 649                  */
 650                 then_action = find_first_action(then_tuple->child->actions, NULL, then->task, node);
 651             } else {
 652                 then_action = find_first_action(then_child->actions, NULL, then->task, node);
 653             }
 654 
 655             if (first_action == NULL) {
 656                 if (is_not_set(first_child->flags, pe_rsc_orphan)
 657                     && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE
 658                     && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) {
 659                     crm_err("Internal error: No action found for %s in %s (first)",
 660                             first_task, first_child->id);
 661 
 662                 } else {
 663                     crm_trace("No action found for %s in %s%s (first)",
 664                               first_task, first_child->id,
 665                               is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
 666                 }
 667                 continue;
 668             }
 669 
 670             /* We're only interested if 'then' is neither stopping nor being demoted */ 
 671             if (then_action == NULL) {
 672                 if (is_not_set(then_child->flags, pe_rsc_orphan)
 673                     && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE
 674                     && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) {
 675                     crm_err("Internal error: No action found for %s in %s (then)",
 676                             then->task, then_child->id);
 677 
 678                 } else {
 679                     crm_trace("No action found for %s in %s%s (then)",
 680                               then->task, then_child->id,
 681                               is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
 682                 }
 683                 continue;
 684             }
 685 
 686             if (order_actions(first_action, then_action, type)) {
 687                 crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x",
 688                           first_action->uuid, is_set(first_action->flags, pe_action_optional),
 689                           then_action->uuid, is_set(then_action->flags, pe_action_optional), type);
 690                 changed |= (pe_graph_updated_first | pe_graph_updated_then);
 691             }
 692             if(first_action && then_action) {
 693                 changed |= then_child->cmds->update_actions(first_action, then_action, node,
 694                                                             first_child->cmds->action_flags(first_action, node),
 695                                                             filter, type);
 696             } else {
 697                 crm_err("Nothing found either for %s (%p) or %s (%p) %s",
 698                         first_child->id, first_action,
 699                         then_child->id, then_action, task2text(task));
 700             }
 701         }
 702     }
 703 
 704     if(children != then->rsc->children) {
 705         g_list_free(children);
 706     }
 707     return changed;
 708 }
 709 
 710 bool can_interleave_actions(pe_action_t *first, pe_action_t *then) 
     /*  */
 711 {
 712     bool interleave = FALSE;
 713     resource_t *rsc = NULL;
 714     const char *interleave_s = NULL;
 715 
 716     if(first->rsc == NULL || then->rsc == NULL) {
 717         crm_trace("Not interleaving %s with %s (both must be resources)", first->uuid, then->uuid);
 718         return FALSE;
 719     } else if(first->rsc == then->rsc) {
 720         crm_trace("Not interleaving %s with %s (must belong to different resources)", first->uuid, then->uuid);
 721         return FALSE;
 722     } else if(first->rsc->variant < pe_clone || then->rsc->variant < pe_clone) {
 723         crm_trace("Not interleaving %s with %s (both sides must be clones, masters, or bundles)", first->uuid, then->uuid);
 724         return FALSE;
 725     }
 726 
 727     if (crm_ends_with(then->uuid, "_stop_0") || crm_ends_with(then->uuid, "_demote_0")) {
 728         rsc = first->rsc;
 729     } else {
 730         rsc = then->rsc;
 731     }
 732 
 733     interleave_s = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERLEAVE);
 734     interleave = crm_is_true(interleave_s);
 735     crm_trace("Interleave %s -> %s: %s (based on %s)",
 736               first->uuid, then->uuid, interleave ? "yes" : "no", rsc->id);
 737 
 738     return interleave;
 739 }
 740 
 741 enum pe_graph_flags
 742 container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
     /*  */
 743                      enum pe_action_flags filter, enum pe_ordering type)
 744 {
 745     enum pe_graph_flags changed = pe_graph_none;
 746 
 747     crm_trace("%s -> %s", first->uuid, then->uuid);
 748 
 749     if(can_interleave_actions(first, then)) {
 750         changed = container_update_interleave_actions(first, then, node, flags, filter, type);
 751 
 752     } else if(then->rsc) {
 753         GListPtr gIter = NULL;
 754         GListPtr children = NULL;
 755 
 756         // Handle the 'primitive' ordering case
 757         changed |= native_update_actions(first, then, node, flags, filter, type);
 758 
 759         // Now any children (or containers in the case of a bundle)
 760         children = get_containers_or_children(then->rsc);
 761         for (gIter = children; gIter != NULL; gIter = gIter->next) {
 762             resource_t *then_child = (resource_t *) gIter->data;
 763             enum pe_graph_flags then_child_changed = pe_graph_none;
 764             action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node);
 765 
 766             if (then_child_action) {
 767                 enum pe_action_flags then_child_flags = then_child->cmds->action_flags(then_child_action, node);
 768 
 769                 if (is_set(then_child_flags, pe_action_runnable)) {
 770                     then_child_changed |=
 771                         then_child->cmds->update_actions(first, then_child_action, node, flags, filter, type);
 772                 }
 773                 changed |= then_child_changed;
 774                 if (then_child_changed & pe_graph_updated_then) {
 775                     for (GListPtr lpc = then_child_action->actions_after; lpc != NULL; lpc = lpc->next) {
 776                         action_wrapper_t *next = (action_wrapper_t *) lpc->data;
 777                         update_action(next->action);
 778                     }
 779                 }
 780             }
 781         }
 782 
 783         if(children != then->rsc->children) {
 784             g_list_free(children);
 785         }
 786     }
 787     return changed;
 788 }
 789 
 790 void
 791 container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
     /*  */
 792 {
 793     container_variant_data_t *container_data = NULL;
 794     get_container_variant_data(container_data, rsc);
 795 
 796     pe_rsc_trace(rsc, "Processing location constraint %s for %s", constraint->id, rsc->id);
 797 
 798     native_rsc_location(rsc, constraint);
 799 
 800     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 801         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 802 
 803         if (tuple->docker) {
 804             tuple->docker->cmds->rsc_location(tuple->docker, constraint);
 805         }
 806         if(tuple->ip) {
 807             tuple->ip->cmds->rsc_location(tuple->ip, constraint);
 808         }
 809     }
 810 
 811     if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) {
 812         container_data->child->cmds->rsc_location(container_data->child, constraint);
 813         container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint);
 814     }
 815 }
 816 
 817 void
 818 container_expand(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
 819 {
 820     container_variant_data_t *container_data = NULL;
 821 
 822     CRM_CHECK(rsc != NULL, return);
 823 
 824     get_container_variant_data(container_data, rsc);
 825 
 826     if(container_data->child) {
 827         container_data->child->cmds->expand(container_data->child, data_set);
 828     }
 829 
 830     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 831         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 832 
 833         CRM_ASSERT(tuple);
 834         if (tuple->remote && tuple->docker && container_fix_remote_addr(tuple->remote)) {
 835             // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside
 836             xmlNode *nvpair = get_xpath_object("//nvpair[@name='addr']", tuple->remote->xml, LOG_ERR);
 837             const char *calculated_addr = container_fix_remote_addr_in(tuple->remote, nvpair, "value");
 838 
 839             if (calculated_addr) {
 840                 crm_trace("Fixed addr for %s on %s", tuple->remote->id, calculated_addr);
 841                 g_hash_table_replace(tuple->remote->parameters, strdup("addr"), strdup(calculated_addr));
 842             } else {
 843                 crm_err("Could not fix addr for %s", tuple->remote->id);
 844             }
 845         }
 846         if(tuple->ip) {
 847             tuple->ip->cmds->expand(tuple->ip, data_set);
 848         }
 849         if(tuple->docker) {
 850             tuple->docker->cmds->expand(tuple->docker, data_set);
 851         }
 852         if(tuple->remote) {
 853             tuple->remote->cmds->expand(tuple->remote, data_set);
 854         }
 855     }
 856 }
 857 
 858 gboolean
 859 container_create_probe(resource_t * rsc, node_t * node, action_t * complete,
     /*  */
 860                    gboolean force, pe_working_set_t * data_set)
 861 {
 862     bool any_created = FALSE;
 863     container_variant_data_t *container_data = NULL;
 864 
 865     CRM_CHECK(rsc != NULL, return FALSE);
 866 
 867     get_container_variant_data(container_data, rsc);
 868     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 869         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 870 
 871         CRM_ASSERT(tuple);
 872         if(tuple->ip) {
 873             any_created |= tuple->ip->cmds->create_probe(tuple->ip, node, complete, force, data_set);
 874         }
 875         if(tuple->child && node->details == tuple->node->details) {
 876             any_created |= tuple->child->cmds->create_probe(tuple->child, node, complete, force, data_set);
 877         }
 878         if(tuple->docker) {
 879             bool created = tuple->docker->cmds->create_probe(tuple->docker, node, complete, force, data_set);
 880 
 881             if(created) {
 882                 any_created = TRUE;
 883                 /* If we're limited to one replica per host (due to
 884                  * the lack of an IP range probably), then we don't
 885                  * want any of our peer containers starting until
 886                  * we've established that no other copies are already
 887                  * running.
 888                  *
 889                  * Partly this is to ensure that replicas_per_host is
 890                  * observed, but also to ensure that the containers
 891                  * don't fail to start because the necessary port
 892                  * mappings (which won't include an IP for uniqueness)
 893                  * are already taken
 894                  */
 895 
 896                 for (GListPtr tIter = container_data->tuples; tIter != NULL && container_data->replicas_per_host == 1; tIter = tIter->next) {
 897                     container_grouping_t *other = (container_grouping_t *)tIter->data;
 898 
 899                     if ((other != tuple) && (other != NULL)
 900                         && (other->docker != NULL)) {
 901 
 902                         custom_action_order(tuple->docker, generate_op_key(tuple->docker->id, RSC_STATUS, 0), NULL,
 903                                             other->docker, generate_op_key(other->docker->id, RSC_START, 0), NULL,
 904                                             pe_order_optional|pe_order_same_node, data_set);
 905                     }
 906                 }
 907             }
 908         }
 909         if (tuple->docker && tuple->remote
 910             && tuple->remote->cmds->create_probe(tuple->remote, node, complete,
 911                                                  force, data_set)) {
 912 
 913             /* Do not probe the remote resource until we know where docker is running
 914              * Required for REMOTE_CONTAINER_HACK to correctly probe remote resources
 915              */
 916             char *probe_uuid = generate_op_key(tuple->remote->id, RSC_STATUS, 0);
 917             action_t *probe = find_first_action(tuple->remote->actions, probe_uuid, NULL, node);
 918 
 919             free(probe_uuid);
 920             if (probe) {
 921                 any_created = TRUE;
 922                 crm_trace("Ordering %s probe on %s",
 923                           tuple->remote->id, node->details->uname);
 924                 custom_action_order(tuple->docker,
 925                                     generate_op_key(tuple->docker->id, RSC_START, 0),
 926                                     NULL, tuple->remote, NULL, probe,
 927                                     pe_order_probe, data_set);
 928             }
 929         }
 930     }
 931     return any_created;
 932 }
 933 
 934 void
 935 container_append_meta(resource_t * rsc, xmlNode * xml)
     /*  */
 936 {
 937 }
 938 
 939 GHashTable *
 940 container_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, const char *attr,
     /*  */
 941                     float factor, enum pe_weights flags)
 942 {
 943     return rsc_merge_weights(rsc, rhs, nodes, attr, factor, flags);
 944 }
 945 
 946 void container_LogActions(
     /*  */
 947     resource_t * rsc, pe_working_set_t * data_set, gboolean terminal)
 948 {
 949     container_variant_data_t *container_data = NULL;
 950 
 951     CRM_CHECK(rsc != NULL, return);
 952 
 953     get_container_variant_data(container_data, rsc);
 954     for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
 955         container_grouping_t *tuple = (container_grouping_t *)gIter->data;
 956 
 957         CRM_ASSERT(tuple);
 958         if(tuple->ip) {
 959             LogActions(tuple->ip, data_set, terminal);
 960         }
 961         if(tuple->docker) {
 962             LogActions(tuple->docker, data_set, terminal);
 963         }
 964         if(tuple->remote) {
 965             LogActions(tuple->remote, data_set, terminal);
 966         }
 967         if(tuple->child) {
 968             LogActions(tuple->child, data_set, terminal);
 969         }
 970     }
 971 }

/* */