root/pengine/allocate.c

/* */

DEFINITIONS

1. update_action_flags
2. check_rsc_parameters
3. CancelXmlOp
4. check_action_definition
5. check_actions_for
6. find_rsc_list
7. check_actions
8. apply_placement_constraints
9. failcount_clear_action_exists
10. check_migration_threshold
11. common_apply_stickiness
12. complex_set_cmds
13. set_alloc_actions
14. calculate_system_health
15. apply_system_health
16. stage0
17. probe_resources
18. rsc_discover_filter
19. stage2
20. stage3
21. stage4
22. sort_rsc_process_order
23. allocate_resources
24. order_start_then_action
25. order_action_then_stop
26. cleanup_orphans
27. stage5
28. is_managed
29. any_managed_resources
30. fence_guest
31. stage6
32. find_actions_by_task
33. rsc_order_then
34. rsc_order_first
35. is_recurring_action
36. apply_container_ordering
37. get_remote_node_state
38. apply_remote_ordering
39. apply_remote_node_ordering
40. order_probes
41. stage7
42. stage8
43. LogNodeActions
44. cleanup_alloc_calculations

   1 /*
   2  * Copyright (C) 2004 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <sys/param.h>
  22 
  23 #include <crm/crm.h>
  24 #include <crm/cib.h>
  25 #include <crm/msg_xml.h>
  26 #include <crm/common/xml.h>
  27 
  28 #include <glib.h>
  29 
  30 #include <crm/pengine/status.h>
  31 #include <pengine.h>
  32 #include <allocate.h>
  33 #include <utils.h>
  34 
  35 CRM_TRACE_INIT_DATA(pe_allocate);
  36 
  37 void set_alloc_actions(pe_working_set_t * data_set);
  38 extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set);
  39 extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
  40 static void apply_remote_node_ordering(pe_working_set_t *data_set);
  41 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
  42 
  43 enum remote_connection_state {
  44     remote_state_unknown = 0,
  45     remote_state_alive = 1,
  46     remote_state_resting = 2,
  47     remote_state_failed = 3,
  48     remote_state_stopped = 4
  49 };
  50 
  51 resource_alloc_functions_t resource_class_alloc_functions[] = {
  52     {
  53      native_merge_weights,
  54      native_color,
  55      native_create_actions,
  56      native_create_probe,
  57      native_internal_constraints,
  58      native_rsc_colocation_lh,
  59      native_rsc_colocation_rh,
  60      native_rsc_location,
  61      native_action_flags,
  62      native_update_actions,
  63      native_expand,
  64      native_append_meta,
  65      },
  66     {
  67      group_merge_weights,
  68      group_color,
  69      group_create_actions,
  70      native_create_probe,
  71      group_internal_constraints,
  72      group_rsc_colocation_lh,
  73      group_rsc_colocation_rh,
  74      group_rsc_location,
  75      group_action_flags,
  76      group_update_actions,
  77      group_expand,
  78      group_append_meta,
  79      },
  80     {
  81      clone_merge_weights,
  82      clone_color,
  83      clone_create_actions,
  84      clone_create_probe,
  85      clone_internal_constraints,
  86      clone_rsc_colocation_lh,
  87      clone_rsc_colocation_rh,
  88      clone_rsc_location,
  89      clone_action_flags,
  90      container_update_actions,
  91      clone_expand,
  92      clone_append_meta,
  93      },
  94     {
  95      master_merge_weights,
  96      master_color,
  97      master_create_actions,
  98      clone_create_probe,
  99      master_internal_constraints,
 100      clone_rsc_colocation_lh,
 101      master_rsc_colocation_rh,
 102      clone_rsc_location,
 103      clone_action_flags,
 104      container_update_actions,
 105      clone_expand,
 106      master_append_meta,
 107      },
 108     {
 109      container_merge_weights,
 110      container_color,
 111      container_create_actions,
 112      container_create_probe,
 113      container_internal_constraints,
 114      container_rsc_colocation_lh,
 115      container_rsc_colocation_rh,
 116      container_rsc_location,
 117      container_action_flags,
 118      container_update_actions,
 119      container_expand,
 120      container_append_meta,
 121      }
 122 };
 123 
 124 gboolean
 125 update_action_flags(action_t * action, enum pe_action_flags flags, const char *source, int line)
     /*  */
 126 {
 127     static unsigned long calls = 0;
 128     gboolean changed = FALSE;
 129     gboolean clear = is_set(flags, pe_action_clear);
 130     enum pe_action_flags last = action->flags;
 131 
 132     if (clear) {
 133         action->flags = crm_clear_bit(source, line, action->uuid, action->flags, flags);
 134     } else {
 135         action->flags = crm_set_bit(source, line, action->uuid, action->flags, flags);
 136     }
 137 
 138     if (last != action->flags) {
 139         calls++;
 140         changed = TRUE;
 141         /* Useful for tracking down _who_ changed a specific flag */
 142         /* CRM_ASSERT(calls != 534); */
 143         clear_bit(flags, pe_action_clear);
 144         crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
 145                   action->uuid, action->node ? action->node->details->uname : "[none]",
 146                   clear ? "un-" : "", flags, last, action->flags, calls, source);
 147     }
 148 
 149     return changed;
 150 }
 151 
 152 static gboolean
 153 check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry,
     /*  */
 154                      gboolean active_here, pe_working_set_t * data_set)
 155 {
 156     int attr_lpc = 0;
 157     gboolean force_restart = FALSE;
 158     gboolean delete_resource = FALSE;
 159     gboolean changed = FALSE;
 160 
 161     const char *value = NULL;
 162     const char *old_value = NULL;
 163 
 164     const char *attr_list[] = {
 165         XML_ATTR_TYPE,
 166         XML_AGENT_ATTR_CLASS,
 167         XML_AGENT_ATTR_PROVIDER
 168     };
 169 
 170     for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
 171         value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
 172         old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
 173         if (value == old_value  /* i.e. NULL */
 174             || crm_str_eq(value, old_value, TRUE)) {
 175             continue;
 176         }
 177 
 178         changed = TRUE;
 179         trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
 180         if (active_here) {
 181             force_restart = TRUE;
 182             crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
 183                        rsc->id, node->details->uname, attr_list[attr_lpc],
 184                        crm_str(old_value), crm_str(value));
 185         }
 186     }
 187     if (force_restart) {
 188         /* make sure the restart happens */
 189         stop_action(rsc, node, FALSE);
 190         set_bit(rsc->flags, pe_rsc_start_pending);
 191         delete_resource = TRUE;
 192 
 193     } else if (changed) {
 194         delete_resource = TRUE;
 195     }
 196     return delete_resource;
 197 }
 198 
 199 static void
 200 CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node,
     /*  */
 201             const char *reason, pe_working_set_t * data_set)
 202 {
 203     int interval = 0;
 204     action_t *cancel = NULL;
 205 
 206     char *key = NULL;
 207     const char *task = NULL;
 208     const char *call_id = NULL;
 209     const char *interval_s = NULL;
 210 
 211     CRM_CHECK(xml_op != NULL, return);
 212     CRM_CHECK(active_node != NULL, return);
 213 
 214     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 215     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 216     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 217 
 218     interval = crm_parse_int(interval_s, "0");
 219 
 220     /* we need to reconstruct the key because of the way we used to construct resource IDs */
 221     key = generate_op_key(rsc->id, task, interval);
 222 
 223     crm_info("Action %s on %s will be stopped: %s",
 224              key, active_node->details->uname, reason ? reason : "unknown");
 225 
 226     /* TODO: This looks highly dangerous if we ever try to schedule 'key' too */
 227     cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set);
 228 
 229     free(cancel->task);
 230     free(cancel->cancel_task);
 231     cancel->task = strdup(RSC_CANCEL);
 232     cancel->cancel_task = strdup(task);
 233 
 234     add_hash_param(cancel->meta, XML_LRM_ATTR_TASK, task);
 235     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
 236     add_hash_param(cancel->meta, XML_LRM_ATTR_INTERVAL, interval_s);
 237 
 238     custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
 239     free(key);
 240     key = NULL;
 241 }
 242 
 243 static gboolean
 244 check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op,
     /*  */
 245                         pe_working_set_t * data_set)
 246 {
 247     char *key = NULL;
 248     int interval = 0;
 249     const char *interval_s = NULL;
 250     const op_digest_cache_t *digest_data = NULL;
 251     gboolean did_change = FALSE;
 252 
 253     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 254     const char *digest_secure = NULL;
 255 
 256     CRM_CHECK(active_node != NULL, return FALSE);
 257     if (safe_str_eq(task, RSC_STOP)) {
 258         return FALSE;
 259     }
 260 
 261     interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL);
 262     interval = crm_parse_int(interval_s, "0");
 263 
 264     if (interval > 0) {
 265         xmlNode *op_match = NULL;
 266 
 267         /* we need to reconstruct the key because of the way we used to construct resource IDs */
 268         key = generate_op_key(rsc->id, task, interval);
 269 
 270         pe_rsc_trace(rsc, "Checking parameters for %s", key);
 271         op_match = find_rsc_op_entry(rsc, key);
 272 
 273         if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
 274             CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
 275             free(key);
 276             return TRUE;
 277 
 278         } else if (op_match == NULL) {
 279             pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
 280             free(key);
 281             return TRUE;
 282         }
 283         free(key);
 284         key = NULL;
 285     }
 286 
 287     crm_trace("Testing %s_%s_%d on %s",
 288               rsc->id, task, interval, active_node->details->uname);
 289     if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
 290         /* Reload based on the start action not a probe */
 291         task = RSC_START;
 292 
 293     } else if (interval == 0 && safe_str_eq(task, RSC_MIGRATED)) {
 294         /* Reload based on the start action not a migrate */
 295         task = RSC_START;
 296     } else if (interval == 0 && safe_str_eq(task, RSC_PROMOTE)) {
 297         /* Reload based on the start action not a promote */
 298         task = RSC_START;
 299     }
 300 
 301     digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
 302 
 303     if(is_set(data_set->flags, pe_flag_sanitized)) {
 304         digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
 305     }
 306 
 307     if(digest_data->rc != RSC_DIGEST_MATCH
 308        && digest_secure
 309        && digest_data->digest_secure_calc
 310        && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
 311         if (is_set(data_set->flags, pe_flag_sanitized)) {
 312             printf("Only 'private' parameters to %s_%s_%d on %s changed: %s\n",
 313                    rsc->id, task, interval, active_node->details->uname,
 314                    crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 315         }
 316 
 317     } else if (digest_data->rc == RSC_DIGEST_RESTART) {
 318         /* Changes that force a restart */
 319         pe_action_t *required = NULL;
 320 
 321         did_change = TRUE;
 322         key = generate_op_key(rsc->id, task, interval);
 323         crm_log_xml_info(digest_data->params_restart, "params:restart");
 324         required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
 325         pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
 326                                   "resource definition change", pe_action_optional, TRUE);
 327 
 328         trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
 329 
 330     } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
 331         /* Changes that can potentially be handled by a reload */
 332         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 333 
 334         did_change = TRUE;
 335         trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
 336         crm_log_xml_info(digest_data->params_all, "params:reload");
 337         key = generate_op_key(rsc->id, task, interval);
 338 
 339         if (interval > 0) {
 340             action_t *op = NULL;
 341 
 342 #if 0
 343             /* Always reload/restart the entire resource */
 344             ReloadRsc(rsc, active_node, data_set);
 345 #else
 346             /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
 347             op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
 348             set_bit(op->flags, pe_action_reschedule);
 349 #endif
 350 
 351         } else if (digest_restart && rsc->isolation_wrapper == NULL && (uber_parent(rsc))->isolation_wrapper == NULL) {
 352             pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
 353 
 354             /* Reload this resource */
 355             ReloadRsc(rsc, active_node, data_set);
 356             free(key);
 357 
 358         } else {
 359             pe_action_t *required = NULL;
 360             pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);
 361 
 362             /* Re-send the start/demote/promote op
 363              * Recurring ops will be detected independently
 364              */
 365             required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
 366             pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
 367                                       "resource definition change", pe_action_optional, TRUE);
 368         }
 369     }
 370 
 371     return did_change;
 372 }
 373 
 374 
 375 static void
 376 check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
     /*  */
 377 {
 378     GListPtr gIter = NULL;
 379     int offset = -1;
 380     int interval = 0;
 381     int stop_index = 0;
 382     int start_index = 0;
 383 
 384     const char *task = NULL;
 385     const char *interval_s = NULL;
 386 
 387     xmlNode *rsc_op = NULL;
 388     GListPtr op_list = NULL;
 389     GListPtr sorted_op_list = NULL;
 390     gboolean is_probe = FALSE;
 391     gboolean did_change = FALSE;
 392 
 393     CRM_CHECK(node != NULL, return);
 394 
 395     if (is_set(rsc->flags, pe_rsc_orphan)) {
 396         resource_t *parent = uber_parent(rsc);
 397         if(parent == NULL
 398            || pe_rsc_is_clone(parent) == FALSE
 399            || is_set(parent->flags, pe_rsc_unique)) {
 400             pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
 401             DeleteRsc(rsc, node, FALSE, data_set);
 402         } else {
 403             pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
 404         }
 405         return;
 406 
 407     } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
 408         if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
 409             DeleteRsc(rsc, node, FALSE, data_set);
 410         }
 411         pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
 412                      rsc->id, node->details->uname);
 413         return;
 414     }
 415 
 416     pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
 417 
 418     if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
 419         DeleteRsc(rsc, node, FALSE, data_set);
 420     }
 421 
 422     for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
 423         if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
 424             op_list = g_list_prepend(op_list, rsc_op);
 425         }
 426     }
 427 
 428     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 429     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 430 
 431     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
 432         xmlNode *rsc_op = (xmlNode *) gIter->data;
 433 
 434         offset++;
 435 
 436         if (start_index < stop_index) {
 437             /* stopped */
 438             continue;
 439         } else if (offset < start_index) {
 440             /* action occurred prior to a start */
 441             continue;
 442         }
 443 
 444         is_probe = FALSE;
 445         did_change = FALSE;
 446         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 447 
 448         interval_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL);
 449         interval = crm_parse_int(interval_s, "0");
 450 
 451         if (interval == 0 && safe_str_eq(task, RSC_STATUS)) {
 452             is_probe = TRUE;
 453         }
 454 
 455         if (interval > 0 &&
 456             (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
 457             CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 458 
 459         } else if (is_probe || safe_str_eq(task, RSC_START) || safe_str_eq(task, RSC_PROMOTE) || interval > 0
 460                    || safe_str_eq(task, RSC_MIGRATED)) {
 461             did_change = check_action_definition(rsc, node, rsc_op, data_set);
 462         }
 463 
 464         if (did_change && pe_get_failcount(node, rsc, NULL, pe_fc_effective,
 465                                            NULL, data_set)) {
 466 
 467             char *key = NULL;
 468             action_t *action_clear = NULL;
 469 
 470             key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
 471             action_clear =
 472                 custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT, node, FALSE, TRUE, data_set);
 473             set_bit(action_clear->flags, pe_action_runnable);
 474 
 475             crm_notice("Clearing failure of %s on %s "
 476                        "because action definition changed " CRM_XS " %s",
 477                        rsc->id, node->details->uname, action_clear->uuid);
 478         }
 479     }
 480 
 481     g_list_free(sorted_op_list);
 482 
 483 }
 484 
 485 static GListPtr
 486 find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones,
     /*  */
 487               gboolean partial, pe_working_set_t * data_set)
 488 {
 489     GListPtr gIter = NULL;
 490     gboolean match = FALSE;
 491 
 492     if (id == NULL) {
 493         return NULL;
 494 
 495     } else if (rsc == NULL && data_set) {
 496 
 497         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
 498             resource_t *child = (resource_t *) gIter->data;
 499 
 500             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
 501         }
 502 
 503         return result;
 504 
 505     } else if (rsc == NULL) {
 506         return NULL;
 507     }
 508 
 509     if (partial) {
 510         if (strstr(rsc->id, id)) {
 511             match = TRUE;
 512 
 513         } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
 514             match = TRUE;
 515         }
 516 
 517     } else {
 518         if (strcmp(rsc->id, id) == 0) {
 519             match = TRUE;
 520 
 521         } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
 522             match = TRUE;
 523         }
 524     }
 525 
 526     if (match) {
 527         result = g_list_prepend(result, rsc);
 528     }
 529 
 530     if (rsc->children) {
 531         gIter = rsc->children;
 532         for (; gIter != NULL; gIter = gIter->next) {
 533             resource_t *child = (resource_t *) gIter->data;
 534 
 535             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
 536         }
 537     }
 538 
 539     return result;
 540 }
 541 
 542 static void
 543 check_actions(pe_working_set_t * data_set)
     /*  */
 544 {
 545     const char *id = NULL;
 546     node_t *node = NULL;
 547     xmlNode *lrm_rscs = NULL;
 548     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 549 
 550     xmlNode *node_state = NULL;
 551 
 552     for (node_state = __xml_first_child(status); node_state != NULL;
 553          node_state = __xml_next_element(node_state)) {
 554         if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
 555             id = crm_element_value(node_state, XML_ATTR_ID);
 556             lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
 557             lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 558 
 559             node = pe_find_node_id(data_set->nodes, id);
 560 
 561             if (node == NULL) {
 562                 continue;
 563 
 564             /* Still need to check actions for a maintenance node to cancel existing monitor operations */
 565             } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
 566                 crm_trace("Skipping param check for %s: can't run resources",
 567                           node->details->uname);
 568                 continue;
 569             }
 570 
 571             crm_trace("Processing node %s", node->details->uname);
 572             if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
 573                 xmlNode *rsc_entry = NULL;
 574 
 575                 for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL;
 576                      rsc_entry = __xml_next_element(rsc_entry)) {
 577                     if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
 578 
 579                         if (xml_has_children(rsc_entry)) {
 580                             GListPtr gIter = NULL;
 581                             GListPtr result = NULL;
 582                             const char *rsc_id = ID(rsc_entry);
 583 
 584                             CRM_CHECK(rsc_id != NULL, return);
 585 
 586                             result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
 587                             for (gIter = result; gIter != NULL; gIter = gIter->next) {
 588                                 resource_t *rsc = (resource_t *) gIter->data;
 589 
 590                                 if (rsc->variant != pe_native) {
 591                                     continue;
 592                                 }
 593                                 check_actions_for(rsc_entry, rsc, node, data_set);
 594                             }
 595                             g_list_free(result);
 596                         }
 597                     }
 598                 }
 599             }
 600         }
 601     }
 602 }
 603 
 604 static gboolean
 605 apply_placement_constraints(pe_working_set_t * data_set)
     /*  */
 606 {
 607     GListPtr gIter = NULL;
 608 
 609     crm_trace("Applying constraints...");
 610 
 611     for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
 612         rsc_to_node_t *cons = (rsc_to_node_t *) gIter->data;
 613 
 614         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
 615     }
 616 
 617     return TRUE;
 618 
 619 }
 620 
 621 static gboolean
 622 failcount_clear_action_exists(node_t * node, resource_t * rsc)
     /*  */
 623 {
 624     gboolean rc = FALSE;
 625     char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
 626     GListPtr list = find_actions_exact(rsc->actions, key, node);
 627 
 628     if (list) {
 629         rc = TRUE;
 630     }
 631     g_list_free(list);
 632     free(key);
 633 
 634     return rc;
 635 }
 636 
 637 /*!
 638  * \internal
 639  * \brief Force resource away if failures hit migration threshold
 640  *
 641  * \param[in,out] rsc       Resource to check for failures
 642  * \param[in,out] node      Node to check for failures
 643  * \param[in,out] data_set  Cluster working set to update
 644  */
 645 static void
 646 check_migration_threshold(resource_t *rsc, node_t *node,
     /*  */
 647                           pe_working_set_t *data_set)
 648 {
 649     int fail_count, countdown;
 650     resource_t *failed;
 651 
 652     /* Migration threshold of 0 means never force away */
 653     if (rsc->migration_threshold == 0) {
 654         return;
 655     }
 656 
 657     // If we're ignoring failures, also ignore the migration threshold
 658     if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
 659         return;
 660     }
 661 
 662     /* If there are no failures, there's no need to force away */
 663     fail_count = pe_get_failcount(node, rsc, NULL,
 664                                   pe_fc_effective|pe_fc_fillers, NULL,
 665                                   data_set);
 666     if (fail_count <= 0) {
 667         return;
 668     }
 669 
 670     /* How many more times recovery will be tried on this node */
 671     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 672 
 673     /* If failed resource has a parent, we'll force the parent away */
 674     failed = rsc;
 675     if (is_not_set(rsc->flags, pe_rsc_unique)) {
 676         failed = uber_parent(rsc);
 677     }
 678 
 679     if (countdown == 0) {
 680         resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
 681         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
 682                  failed->id, node->details->uname, fail_count,
 683                  rsc->migration_threshold);
 684     } else {
 685         crm_info("%s can fail %d more times on %s before being forced off",
 686                  failed->id, countdown, node->details->uname);
 687     }
 688 }
 689 
 690 static void
 691 common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
     /*  */
 692 {
 693     if (rsc->children) {
 694         GListPtr gIter = rsc->children;
 695 
 696         for (; gIter != NULL; gIter = gIter->next) {
 697             resource_t *child_rsc = (resource_t *) gIter->data;
 698 
 699             common_apply_stickiness(child_rsc, node, data_set);
 700         }
 701         return;
 702     }
 703 
 704     if (is_set(rsc->flags, pe_rsc_managed)
 705         && rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) {
 706         node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
 707         node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 708 
 709         if (current == NULL) {
 710 
 711         } else if (match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
 712             resource_t *sticky_rsc = rsc;
 713 
 714             resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
 715             pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
 716                          " (node=%s, weight=%d)", sticky_rsc->id,
 717                          node->details->uname, rsc->stickiness);
 718         } else {
 719             GHashTableIter iter;
 720             node_t *nIter = NULL;
 721 
 722             pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
 723                          " and node %s is not explicitly allowed", rsc->id, node->details->uname);
 724             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
 725             while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
 726                 crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
 727             }
 728         }
 729     }
 730 
 731     /* Check the migration threshold only if a failcount clear action
 732      * has not already been placed for this resource on the node.
 733      * There is no sense in potentially forcing the resource from this
 734      * node if the failcount is being reset anyway. */
 735     if (failcount_clear_action_exists(node, rsc) == FALSE) {
 736         check_migration_threshold(rsc, node, data_set);
 737     }
 738 }
 739 
 740 void
 741 complex_set_cmds(resource_t * rsc)
     /*  */
 742 {
 743     GListPtr gIter = rsc->children;
 744 
 745     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
 746 
 747     for (; gIter != NULL; gIter = gIter->next) {
 748         resource_t *child_rsc = (resource_t *) gIter->data;
 749 
 750         complex_set_cmds(child_rsc);
 751     }
 752 }
 753 
 754 void
 755 set_alloc_actions(pe_working_set_t * data_set)
     /*  */
 756 {
 757 
 758     GListPtr gIter = data_set->resources;
 759 
 760     for (; gIter != NULL; gIter = gIter->next) {
 761         resource_t *rsc = (resource_t *) gIter->data;
 762 
 763         complex_set_cmds(rsc);
 764     }
 765 }
 766 
 767 static void
 768 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
     /*  */
 769 {
 770     const char *key = (const char *)gKey;
 771     const char *value = (const char *)gValue;
 772     int *system_health = (int *)user_data;
 773 
 774     if (!gKey || !gValue || !user_data) {
 775         return;
 776     }
 777 
 778     if (crm_starts_with(key, "#health")) {
 779         int score;
 780 
 781         /* Convert the value into an integer */
 782         score = char2score(value);
 783 
 784         /* Add it to the running total */
 785         *system_health = merge_weights(score, *system_health);
 786     }
 787 }
 788 
 789 static gboolean
 790 apply_system_health(pe_working_set_t * data_set)
     /*  */
 791 {
 792     GListPtr gIter = NULL;
 793     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
 794     int base_health = 0;
 795 
 796     if (health_strategy == NULL || safe_str_eq(health_strategy, "none")) {
 797         /* Prevent any accidental health -> score translation */
 798         node_score_red = 0;
 799         node_score_yellow = 0;
 800         node_score_green = 0;
 801         return TRUE;
 802 
 803     } else if (safe_str_eq(health_strategy, "migrate-on-red")) {
 804 
 805         /* Resources on nodes which have health values of red are
 806          * weighted away from that node.
 807          */
 808         node_score_red = -INFINITY;
 809         node_score_yellow = 0;
 810         node_score_green = 0;
 811 
 812     } else if (safe_str_eq(health_strategy, "only-green")) {
 813 
 814         /* Resources on nodes which have health values of red or yellow
 815          * are forced away from that node.
 816          */
 817         node_score_red = -INFINITY;
 818         node_score_yellow = -INFINITY;
 819         node_score_green = 0;
 820 
 821     } else if (safe_str_eq(health_strategy, "progressive")) {
 822         /* Same as the above, but use the r/y/g scores provided by the user
 823          * Defaults are provided by the pe_prefs table
 824          * Also, custom health "base score" can be used
 825          */
 826         base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0");
 827 
 828     } else if (safe_str_eq(health_strategy, "custom")) {
 829 
 830         /* Requires the admin to configure the rsc_location constaints for
 831          * processing the stored health scores
 832          */
 833         /* TODO: Check for the existence of appropriate node health constraints */
 834         return TRUE;
 835 
 836     } else {
 837         crm_err("Unknown node health strategy: %s", health_strategy);
 838         return FALSE;
 839     }
 840 
 841     crm_info("Applying automated node health strategy: %s", health_strategy);
 842 
 843     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
 844         int system_health = base_health;
 845         node_t *node = (node_t *) gIter->data;
 846 
 847         /* Search through the node hash table for system health entries. */
 848         g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
 849 
 850         crm_info(" Node %s has an combined system health of %d",
 851                  node->details->uname, system_health);
 852 
 853         /* If the health is non-zero, then create a new rsc2node so that the
 854          * weight will be added later on.
 855          */
 856         if (system_health != 0) {
 857 
 858             GListPtr gIter2 = data_set->resources;
 859 
 860             for (; gIter2 != NULL; gIter2 = gIter2->next) {
 861                 resource_t *rsc = (resource_t *) gIter2->data;
 862 
 863                 rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
 864             }
 865         }
 866     }
 867 
 868     return TRUE;
 869 }
 870 
 871 gboolean
 872 stage0(pe_working_set_t * data_set)
     /*  */
 873 {
 874     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 875 
 876     if (data_set->input == NULL) {
 877         return FALSE;
 878     }
 879 
 880     if (is_set(data_set->flags, pe_flag_have_status) == FALSE) {
 881         crm_trace("Calculating status");
 882         cluster_status(data_set);
 883     }
 884 
 885     set_alloc_actions(data_set);
 886     apply_system_health(data_set);
 887     unpack_constraints(cib_constraints, data_set);
 888 
 889     return TRUE;
 890 }
 891 
 892 /*
 893  * Check nodes for resources started outside of the LRM
 894  */
 895 gboolean
 896 probe_resources(pe_working_set_t * data_set)
     /*  */
 897 {
 898     action_t *probe_node_complete = NULL;
 899 
 900     for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
 901         node_t *node = (node_t *) gIter->data;
 902         const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
 903 
 904         if (node->details->online == FALSE) {
 905 
 906             if (is_baremetal_remote_node(node) && node->details->remote_rsc
 907                 && (get_remote_node_state(node) == remote_state_failed)) {
 908 
 909                 pe_fence_node(data_set, node, "the connection is unrecoverable");
 910             }
 911             continue;
 912 
 913         } else if (node->details->unclean) {
 914             continue;
 915 
 916         } else if (node->details->rsc_discovery_enabled == FALSE) {
 917             /* resource discovery is disabled for this node */
 918             continue;
 919         }
 920 
 921         if (probed != NULL && crm_is_true(probed) == FALSE) {
 922             action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
 923                                                CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
 924 
 925             add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 926             continue;
 927         }
 928 
 929         for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
 930             resource_t *rsc = (resource_t *) gIter2->data;
 931 
 932             rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
 933         }
 934     }
 935     return TRUE;
 936 }
 937 
 938 static void
 939 rsc_discover_filter(resource_t *rsc, node_t *node)
     /*  */
 940 {
 941     GListPtr gIter = rsc->children;
 942     resource_t *top = uber_parent(rsc);
 943     node_t *match;
 944 
 945     if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
 946         return;
 947     }
 948 
 949     for (; gIter != NULL; gIter = gIter->next) {
 950         resource_t *child_rsc = (resource_t *) gIter->data;
 951         rsc_discover_filter(child_rsc, node);
 952     }
 953 
 954     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 955     if (match && match->rsc_discover_mode != pe_discover_exclusive) {
 956         match->weight = -INFINITY;
 957     }
 958 }
 959 
 960 /*
 961  * Count how many valid nodes we have (so we know the maximum number of
 962  *  colors we can resolve).
 963  *
 964  * Apply node constraints (i.e. filter the "allowed_nodes" part of resources)
 965  */
 966 gboolean
 967 stage2(pe_working_set_t * data_set)
     /*  */
 968 {
 969     GListPtr gIter = NULL;
 970 
 971     crm_trace("Applying placement constraints");
 972 
 973     gIter = data_set->nodes;
 974     for (; gIter != NULL; gIter = gIter->next) {
 975         node_t *node = (node_t *) gIter->data;
 976 
 977         if (node == NULL) {
 978             /* error */
 979 
 980         } else if (node->weight >= 0.0  /* global weight */
 981                    && node->details->online && node->details->type != node_ping) {
 982             data_set->max_valid_nodes++;
 983         }
 984     }
 985 
 986     apply_placement_constraints(data_set);
 987 
 988     gIter = data_set->nodes;
 989     for (; gIter != NULL; gIter = gIter->next) {
 990         GListPtr gIter2 = NULL;
 991         node_t *node = (node_t *) gIter->data;
 992 
 993         gIter2 = data_set->resources;
 994         for (; gIter2 != NULL; gIter2 = gIter2->next) {
 995             resource_t *rsc = (resource_t *) gIter2->data;
 996 
 997             common_apply_stickiness(rsc, node, data_set);
 998             rsc_discover_filter(rsc, node);
 999         }
1000     }
1001 
1002     return TRUE;
1003 }
1004 
1005 /*
1006  * Create internal resource constraints before allocation
1007  */
1008 gboolean
1009 stage3(pe_working_set_t * data_set)
     /*  */
1010 {
1011 
1012     GListPtr gIter = data_set->resources;
1013 
1014     for (; gIter != NULL; gIter = gIter->next) {
1015         resource_t *rsc = (resource_t *) gIter->data;
1016 
1017         rsc->cmds->internal_constraints(rsc, data_set);
1018     }
1019 
1020     return TRUE;
1021 }
1022 
1023 /*
1024  * Check for orphaned or redefined actions
1025  */
1026 gboolean
1027 stage4(pe_working_set_t * data_set)
     /*  */
1028 {
1029     check_actions(data_set);
1030     return TRUE;
1031 }
1032 
1033 static gint
1034 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
     /*  */
1035 {
1036     int rc = 0;
1037     int r1_weight = -INFINITY;
1038     int r2_weight = -INFINITY;
1039 
1040     const char *reason = "existence";
1041 
1042     const GListPtr nodes = (GListPtr) data;
1043     resource_t *resource1 = (resource_t *) convert_const_pointer(a);
1044     resource_t *resource2 = (resource_t *) convert_const_pointer(b);
1045 
1046     node_t *r1_node = NULL;
1047     node_t *r2_node = NULL;
1048     GListPtr gIter = NULL;
1049     GHashTable *r1_nodes = NULL;
1050     GHashTable *r2_nodes = NULL;
1051 
1052     if (a == NULL && b == NULL) {
1053         goto done;
1054     }
1055     if (a == NULL) {
1056         return 1;
1057     }
1058     if (b == NULL) {
1059         return -1;
1060     }
1061 
1062     reason = "priority";
1063     r1_weight = resource1->priority;
1064     r2_weight = resource2->priority;
1065 
1066     if (r1_weight > r2_weight) {
1067         rc = -1;
1068         goto done;
1069     }
1070 
1071     if (r1_weight < r2_weight) {
1072         rc = 1;
1073         goto done;
1074     }
1075 
1076     reason = "no node list";
1077     if (nodes == NULL) {
1078         goto done;
1079     }
1080 
1081     r1_nodes =
1082         rsc_merge_weights(resource1, resource1->id, NULL, NULL, 1,
1083                           pe_weights_forward | pe_weights_init);
1084     dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes);
1085     r2_nodes =
1086         rsc_merge_weights(resource2, resource2->id, NULL, NULL, 1,
1087                           pe_weights_forward | pe_weights_init);
1088     dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes);
1089 
1090     /* Current location score */
1091     reason = "current location";
1092     r1_weight = -INFINITY;
1093     r2_weight = -INFINITY;
1094 
1095     if (resource1->running_on) {
1096         r1_node = g_list_nth_data(resource1->running_on, 0);
1097         r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
1098         if (r1_node != NULL) {
1099             r1_weight = r1_node->weight;
1100         }
1101     }
1102     if (resource2->running_on) {
1103         r2_node = g_list_nth_data(resource2->running_on, 0);
1104         r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
1105         if (r2_node != NULL) {
1106             r2_weight = r2_node->weight;
1107         }
1108     }
1109 
1110     if (r1_weight > r2_weight) {
1111         rc = -1;
1112         goto done;
1113     }
1114 
1115     if (r1_weight < r2_weight) {
1116         rc = 1;
1117         goto done;
1118     }
1119 
1120     reason = "score";
1121     for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
1122         node_t *node = (node_t *) gIter->data;
1123 
1124         r1_node = NULL;
1125         r2_node = NULL;
1126 
1127         r1_weight = -INFINITY;
1128         if (r1_nodes) {
1129             r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
1130         }
1131         if (r1_node) {
1132             r1_weight = r1_node->weight;
1133         }
1134 
1135         r2_weight = -INFINITY;
1136         if (r2_nodes) {
1137             r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
1138         }
1139         if (r2_node) {
1140             r2_weight = r2_node->weight;
1141         }
1142 
1143         if (r1_weight > r2_weight) {
1144             rc = -1;
1145             goto done;
1146         }
1147 
1148         if (r1_weight < r2_weight) {
1149             rc = 1;
1150             goto done;
1151         }
1152     }
1153 
1154   done:
1155     crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
1156               resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
1157               rc < 0 ? '>' : rc > 0 ? '<' : '=',
1158               resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
1159 
1160     if (r1_nodes) {
1161         g_hash_table_destroy(r1_nodes);
1162     }
1163     if (r2_nodes) {
1164         g_hash_table_destroy(r2_nodes);
1165     }
1166 
1167     return rc;
1168 }
1169 
1170 static void
1171 allocate_resources(pe_working_set_t * data_set)
     /*  */
1172 {
1173     GListPtr gIter = NULL;
1174 
1175     if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {
1176         /* Force remote connection resources to be allocated first. This
1177          * also forces any colocation dependencies to be allocated as well */
1178         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1179             resource_t *rsc = (resource_t *) gIter->data;
1180             if (rsc->is_remote_node == FALSE) {
1181                 continue;
1182             }
1183             pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
1184             /* For remote node connection resources, always prefer the partial
1185              * migration target during resource allocation, if the rsc is in the
1186              * middle of a migration.
1187              */
1188             rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
1189         }
1190     }
1191 
1192     /* now do the rest of the resources */
1193     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1194         resource_t *rsc = (resource_t *) gIter->data;
1195         if (rsc->is_remote_node == TRUE) {
1196             continue;
1197         }
1198         pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
1199         rsc->cmds->allocate(rsc, NULL, data_set);
1200     }
1201 }
1202 
1203 /* We always use pe_order_preserve with these convenience functions to exempt
1204  * internally generated constraints from the prohibition of user constraints
1205  * involving remote connection resources.
1206  *
1207  * The start ordering additionally uses pe_order_runnable_left so that the
1208  * specified action is not runnable if the start is not runnable.
1209  */
1210 
1211 static inline void
1212 order_start_then_action(resource_t *lh_rsc, action_t *rh_action,
     /*  */
1213                         enum pe_ordering extra, pe_working_set_t *data_set)
1214 {
1215     if (lh_rsc && rh_action && data_set) {
1216         custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
1217                             rh_action->rsc, NULL, rh_action,
1218                             pe_order_preserve | pe_order_runnable_left | extra,
1219                             data_set);
1220     }
1221 }
1222 
1223 static inline void
1224 order_action_then_stop(action_t *lh_action, resource_t *rh_rsc,
     /*  */
1225                        enum pe_ordering extra, pe_working_set_t *data_set)
1226 {
1227     if (lh_action && rh_rsc && data_set) {
1228         custom_action_order(lh_action->rsc, NULL, lh_action,
1229                             rh_rsc, stop_key(rh_rsc), NULL,
1230                             pe_order_preserve | extra, data_set);
1231     }
1232 }
1233 
1234 static void
1235 cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
     /*  */
1236 {
1237     GListPtr gIter = NULL;
1238 
1239     if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
1240         return;
1241     }
1242 
1243     /* Don't recurse into ->children, those are just unallocated clone instances */
1244     if(is_not_set(rsc->flags, pe_rsc_orphan)) {
1245         return;
1246     }
1247 
1248     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1249         node_t *node = (node_t *) gIter->data;
1250 
1251         if (node->details->online
1252             && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
1253                                 data_set)) {
1254 
1255             char *key = generate_op_key(rsc->id, CRM_OP_CLEAR_FAILCOUNT, 0);
1256             action_t *clear_op = custom_action(rsc, key, CRM_OP_CLEAR_FAILCOUNT,
1257                                                node, FALSE, TRUE, data_set);
1258 
1259             add_hash_param(clear_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
1260 
1261             pe_rsc_info(rsc,
1262                         "Clearing failure of %s on %s because it is orphaned "
1263                         CRM_XS " %s",
1264                         rsc->id, node->details->uname, clear_op->uuid);
1265 
1266             /* We can't use order_action_then_stop() here because its
1267              * pe_order_preserve breaks things
1268              */
1269             custom_action_order(clear_op->rsc, NULL, clear_op,
1270                                 rsc, stop_key(rsc), NULL,
1271                                 pe_order_optional, data_set);
1272         }
1273     }
1274 }
1275 
1276 gboolean
1277 stage5(pe_working_set_t * data_set)
     /*  */
1278 {
1279     GListPtr gIter = NULL;
1280 
1281     if (safe_str_neq(data_set->placement_strategy, "default")) {
1282         GListPtr nodes = g_list_copy(data_set->nodes);
1283 
1284         nodes = g_list_sort_with_data(nodes, sort_node_weight, NULL);
1285 
1286         data_set->resources =
1287             g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
1288 
1289         g_list_free(nodes);
1290     }
1291 
1292     gIter = data_set->nodes;
1293     for (; gIter != NULL; gIter = gIter->next) {
1294         node_t *node = (node_t *) gIter->data;
1295 
1296         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Original", node);
1297     }
1298 
1299     crm_trace("Allocating services");
1300     /* Take (next) highest resource, assign it and create its actions */
1301 
1302     allocate_resources(data_set);
1303 
1304     gIter = data_set->nodes;
1305     for (; gIter != NULL; gIter = gIter->next) {
1306         node_t *node = (node_t *) gIter->data;
1307 
1308         dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node);
1309     }
1310 
1311     if (is_set(data_set->flags, pe_flag_startup_probes)) {
1312         crm_trace("Calculating needed probes");
1313         /* This code probably needs optimization
1314          * ptest -x with 100 nodes, 100 clones and clone-max=100:
1315 
1316          With probes:
1317 
1318          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1319          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1320          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1321          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
1322          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1323          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
1324          ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
1325          ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
1326          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
1327          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1328          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1329          36s
1330          ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1331 
1332          Without probes:
1333 
1334          ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1335          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1336          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1337          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
1338          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1339          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
1340          ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
1341          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
1342          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1343          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1344          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1345         */
1346 
1347         probe_resources(data_set);
1348     }
1349 
1350     crm_trace("Handle orphans");
1351 
1352     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1353         resource_t *rsc = (resource_t *) gIter->data;
1354         cleanup_orphans(rsc, data_set);
1355     }
1356 
1357     crm_trace("Creating actions");
1358 
1359     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1360         resource_t *rsc = (resource_t *) gIter->data;
1361 
1362         rsc->cmds->create_actions(rsc, data_set);
1363     }
1364 
1365     crm_trace("Creating done");
1366     return TRUE;
1367 }
1368 
1369 static gboolean
1370 is_managed(const resource_t * rsc)
     /*  */
1371 {
1372     GListPtr gIter = rsc->children;
1373 
1374     if (is_set(rsc->flags, pe_rsc_managed)) {
1375         return TRUE;
1376     }
1377 
1378     for (; gIter != NULL; gIter = gIter->next) {
1379         resource_t *child_rsc = (resource_t *) gIter->data;
1380 
1381         if (is_managed(child_rsc)) {
1382             return TRUE;
1383         }
1384     }
1385 
1386     return FALSE;
1387 }
1388 
1389 static gboolean
1390 any_managed_resources(pe_working_set_t * data_set)
     /*  */
1391 {
1392 
1393     GListPtr gIter = data_set->resources;
1394 
1395     for (; gIter != NULL; gIter = gIter->next) {
1396         resource_t *rsc = (resource_t *) gIter->data;
1397 
1398         if (is_managed(rsc)) {
1399             return TRUE;
1400         }
1401     }
1402     return FALSE;
1403 }
1404 
1405 /*!
1406  * \internal
1407  * \brief Create pseudo-op for guest node fence, and order relative to it
1408  *
1409  * \param[in] node      Guest node to fence
1410  * \param[in] done      STONITH_DONE operation
1411  * \param[in] data_set  Working set of CIB state
1412  */
1413 static void
1414 fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set)
     /*  */
1415 {
1416     resource_t *container = node->details->remote_rsc->container;
1417     pe_action_t *stop = NULL;
1418     pe_action_t *stonith_op = NULL;
1419 
1420     /* The fence action is just a label; we don't do anything differently for
1421      * off vs. reboot. We specify it explicitly, rather than let it default to
1422      * cluster's default action, because we are not _initiating_ fencing -- we
1423      * are creating a pseudo-event to describe fencing that is already occurring
1424      * by other means (container recovery).
1425      */
1426     const char *fence_action = "off";
1427 
1428     /* Check whether guest's container resource is has any explicit stop or
1429      * start (the stop may be implied by fencing of the guest's host).
1430      */
1431     if (container) {
1432         stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
1433 
1434         if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
1435             fence_action = "reboot";
1436         }
1437     }
1438 
1439     /* Create a fence pseudo-event, so we have an event to order actions
1440      * against, and crmd can always detect it.
1441      */
1442     stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set);
1443     update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable,
1444                         __FUNCTION__, __LINE__);
1445 
1446     /* We want to imply stops/demotes after the guest is stopped, not wait until
1447      * it is restarted, so we always order pseudo-fencing after stop, not start
1448      * (even though start might be closer to what is done for a real reboot).
1449      */
1450     if(stop && is_set(stop->flags, pe_action_pseudo)) {
1451         pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set);
1452         crm_info("Implying guest node %s is down (action %d) after %s fencing",
1453                  node->details->uname, stonith_op->id, stop->node->details->uname);
1454         order_actions(parent_stonith_op, stonith_op,
1455                       pe_order_runnable_left|pe_order_implies_then);
1456 
1457     } else if (stop) {
1458         order_actions(stop, stonith_op,
1459                       pe_order_runnable_left|pe_order_implies_then);
1460         crm_info("Implying guest node %s is down (action %d) "
1461                  "after container %s is stopped (action %d)",
1462                  node->details->uname, stonith_op->id,
1463                  container->id, stop->id);
1464     } else {
1465         crm_info("Implying guest node %s is down (action %d) ",
1466                  node->details->uname, stonith_op->id);
1467     }
1468 
1469     /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */
1470 
1471     /* Order/imply other actions relative to pseudo-fence as with real fence */
1472     stonith_constraints(node, stonith_op, data_set);
1473     order_actions(stonith_op, done, pe_order_implies_then);
1474 }
1475 
1476 /*
1477  * Create dependencies for stonith and shutdown operations
1478  */
1479 gboolean
1480 stage6(pe_working_set_t * data_set)
     /*  */
1481 {
1482     action_t *dc_down = NULL;
1483     action_t *dc_fence = NULL;
1484     action_t *stonith_op = NULL;
1485     action_t *last_stonith = NULL;
1486     gboolean integrity_lost = FALSE;
1487     action_t *all_stopped = get_pseudo_op(ALL_STOPPED, data_set);
1488     action_t *done = get_pseudo_op(STONITH_DONE, data_set);
1489     gboolean need_stonith = TRUE;
1490     GListPtr gIter;
1491     GListPtr stonith_ops = NULL;
1492 
1493     /* Remote ordering constraints need to happen prior to calculate
1494      * fencing because it is one more place we will mark the node as
1495      * dirty.
1496      *
1497      * A nice side-effect of doing it first is that we can remove a
1498      * bunch of special logic from apply_*_ordering() because its
1499      * already part of pe_fence_node()
1500      */
1501     crm_trace("Creating remote ordering constraints");
1502     apply_remote_node_ordering(data_set);
1503 
1504     crm_trace("Processing fencing and shutdown cases");
1505     if (any_managed_resources(data_set) == FALSE) {
1506         crm_notice("Delaying fencing operations until there are resources to manage");
1507         need_stonith = FALSE;
1508     }
1509 
1510     /* Check each node for stonith/shutdown */
1511     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1512         node_t *node = (node_t *) gIter->data;
1513 
1514         /* Guest nodes are "fenced" by recovering their container resource,
1515          * so handle them separately.
1516          */
1517         if (is_container_remote_node(node)) {
1518             if (node->details->remote_requires_reset && need_stonith) {
1519                 fence_guest(node, done, data_set);
1520             }
1521             continue;
1522         }
1523 
1524         stonith_op = NULL;
1525 
1526         if (node->details->unclean
1527             && need_stonith && pe_can_fence(data_set, node)) {
1528 
1529             stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set);
1530             pe_warn("Scheduling Node %s for STONITH", node->details->uname);
1531 
1532             stonith_constraints(node, stonith_op, data_set);
1533 
1534             if (node->details->is_dc) {
1535                 dc_down = stonith_op;
1536                 dc_fence = stonith_op;
1537 
1538             } else if (is_set(data_set->flags, pe_flag_concurrent_fencing) == FALSE) {
1539                 if (last_stonith) {
1540                     order_actions(last_stonith, stonith_op, pe_order_optional);
1541                 }
1542                 last_stonith = stonith_op;
1543 
1544             } else {
1545                 order_actions(stonith_op, done, pe_order_implies_then);
1546                 stonith_ops = g_list_append(stonith_ops, stonith_op);
1547             }
1548 
1549         } else if (node->details->online && node->details->shutdown &&
1550                 /* TODO define what a shutdown op means for a remote node.
1551                  * For now we do not send shutdown operations for remote nodes, but
1552                  * if we can come up with a good use for this in the future, we will. */
1553                     is_remote_node(node) == FALSE) {
1554 
1555             action_t *down_op = NULL;
1556 
1557             crm_notice("Scheduling Node %s for shutdown", node->details->uname);
1558 
1559             down_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname),
1560                                     CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set);
1561 
1562             shutdown_constraints(node, down_op, data_set);
1563             add_hash_param(down_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
1564 
1565             if (node->details->is_dc) {
1566                 dc_down = down_op;
1567             }
1568         }
1569 
1570         if (node->details->unclean && stonith_op == NULL) {
1571             integrity_lost = TRUE;
1572             pe_warn("Node %s is unclean!", node->details->uname);
1573         }
1574     }
1575 
1576     if (integrity_lost) {
1577         if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
1578             pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
1579             pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
1580 
1581         } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
1582             crm_notice("Cannot fence unclean nodes until quorum is"
1583                        " attained (or no-quorum-policy is set to ignore)");
1584         }
1585     }
1586 
1587     if (dc_down != NULL) {
1588         GListPtr gIter = NULL;
1589 
1590         crm_trace("Ordering shutdowns before %s on %s (DC)",
1591                   dc_down->task, dc_down->node->details->uname);
1592 
1593         add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
1594 
1595         for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
1596             action_t *node_stop = (action_t *) gIter->data;
1597 
1598             if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) {
1599                 continue;
1600             } else if (node_stop->node->details->is_dc) {
1601                 continue;
1602             }
1603 
1604             crm_debug("Ordering shutdown on %s before %s on %s",
1605                       node_stop->node->details->uname,
1606                       dc_down->task, dc_down->node->details->uname);
1607 
1608             order_actions(node_stop, dc_down, pe_order_optional);
1609         }
1610 
1611         if (last_stonith) {
1612             if (dc_down != last_stonith) {
1613                 order_actions(last_stonith, dc_down, pe_order_optional);
1614             }
1615 
1616         } else {
1617             GListPtr gIter2 = NULL;
1618 
1619             for (gIter2 = stonith_ops; gIter2 != NULL; gIter2 = gIter2->next) {
1620                 stonith_op = (action_t *) gIter2->data;
1621 
1622                 if (dc_down != stonith_op) {
1623                     order_actions(stonith_op, dc_down, pe_order_optional);
1624                 }
1625             }
1626         }
1627     }
1628 
1629 
1630     if (dc_fence) {
1631         order_actions(dc_down, done, pe_order_implies_then);
1632 
1633     } else if (last_stonith) {
1634         order_actions(last_stonith, done, pe_order_implies_then);
1635     }
1636 
1637     order_actions(done, all_stopped, pe_order_implies_then);
1638 
1639     g_list_free(stonith_ops);
1640     return TRUE;
1641 }
1642 
1643 /*
1644  * Determine the sets of independent actions and the correct order for the
1645  *  actions in each set.
1646  *
1647  * Mark dependencies of un-runnable actions un-runnable
1648  *
1649  */
1650 static GListPtr
1651 find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key)
     /*  */
1652 {
1653     GListPtr list = NULL;
1654 
1655     list = find_actions(actions, original_key, NULL);
1656     if (list == NULL) {
1657         /* we're potentially searching a child of the original resource */
1658         char *key = NULL;
1659         char *tmp = NULL;
1660         char *task = NULL;
1661         int interval = 0;
1662 
1663         if (parse_op_key(original_key, &tmp, &task, &interval)) {
1664             key = generate_op_key(rsc->id, task, interval);
1665             /* crm_err("looking up %s instead of %s", key, original_key); */
1666             /* slist_iter(action, action_t, actions, lpc, */
1667             /*         crm_err("  - %s", action->uuid)); */
1668             list = find_actions(actions, key, NULL);
1669 
1670         } else {
1671             crm_err("search key: %s", original_key);
1672         }
1673 
1674         free(key);
1675         free(tmp);
1676         free(task);
1677     }
1678 
1679     return list;
1680 }
1681 
1682 static void
1683 rsc_order_then(action_t * lh_action, resource_t * rsc, order_constraint_t * order)
     /*  */
1684 {
1685     GListPtr gIter = NULL;
1686     GListPtr rh_actions = NULL;
1687     action_t *rh_action = NULL;
1688     enum pe_ordering type = order->type;
1689 
1690     CRM_CHECK(rsc != NULL, return);
1691     CRM_CHECK(order != NULL, return);
1692 
1693     rh_action = order->rh_action;
1694     crm_trace("Processing RH of ordering constraint %d", order->id);
1695 
1696     if (rh_action != NULL) {
1697         rh_actions = g_list_prepend(NULL, rh_action);
1698 
1699     } else if (rsc != NULL) {
1700         rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
1701     }
1702 
1703     if (rh_actions == NULL) {
1704         pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
1705                      " ignoring", rsc->id, order->rh_action_task);
1706         if (lh_action) {
1707             pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
1708         }
1709         return;
1710     }
1711 
1712     if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) {
1713         pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
1714                      order->rh_action_task);
1715         clear_bit(type, pe_order_implies_then);
1716     }
1717 
1718     gIter = rh_actions;
1719     for (; gIter != NULL; gIter = gIter->next) {
1720         action_t *rh_action_iter = (action_t *) gIter->data;
1721 
1722         if (lh_action) {
1723             order_actions(lh_action, rh_action_iter, type);
1724 
1725         } else if (type & pe_order_implies_then) {
1726             update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
1727             crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
1728         } else {
1729             crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
1730         }
1731     }
1732 
1733     g_list_free(rh_actions);
1734 }
1735 
1736 static void
1737 rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_t * data_set)
     /*  */
1738 {
1739     GListPtr gIter = NULL;
1740     GListPtr lh_actions = NULL;
1741     action_t *lh_action = order->lh_action;
1742     resource_t *rh_rsc = order->rh_rsc;
1743 
1744     crm_trace("Processing LH of ordering constraint %d", order->id);
1745     CRM_ASSERT(lh_rsc != NULL);
1746 
1747     if (lh_action != NULL) {
1748         lh_actions = g_list_prepend(NULL, lh_action);
1749 
1750     } else if (lh_action == NULL) {
1751         lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
1752     }
1753 
1754     if (lh_actions == NULL && lh_rsc != rh_rsc) {
1755         char *key = NULL;
1756         char *rsc_id = NULL;
1757         char *op_type = NULL;
1758         int interval = 0;
1759 
1760         parse_op_key(order->lh_action_task, &rsc_id, &op_type, &interval);
1761         key = generate_op_key(lh_rsc->id, op_type, interval);
1762 
1763         if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) {
1764             free(key);
1765             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1766                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1767 
1768         } else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) {
1769             free(key);
1770             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1771                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1772 
1773         } else {
1774             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
1775                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1776             lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
1777             lh_actions = g_list_prepend(NULL, lh_action);
1778         }
1779 
1780         free(op_type);
1781         free(rsc_id);
1782     }
1783 
1784     gIter = lh_actions;
1785     for (; gIter != NULL; gIter = gIter->next) {
1786         action_t *lh_action_iter = (action_t *) gIter->data;
1787 
1788         if (rh_rsc == NULL && order->rh_action) {
1789             rh_rsc = order->rh_action->rsc;
1790         }
1791         if (rh_rsc) {
1792             rsc_order_then(lh_action_iter, rh_rsc, order);
1793 
1794         } else if (order->rh_action) {
1795             order_actions(lh_action_iter, order->rh_action, order->type);
1796         }
1797     }
1798 
1799     g_list_free(lh_actions);
1800 }
1801 
1802 extern gboolean update_action(action_t * action);
1803 extern void update_colo_start_chain(action_t * action);
1804 
1805 static int
1806 is_recurring_action(action_t *action) 
     /*  */
1807 {
1808     const char *interval_s = g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL);
1809     int interval = crm_parse_int(interval_s, "0");
1810     if(interval > 0) {
1811         return TRUE;
1812     }
1813     return FALSE;
1814 }
1815 
1816 static void
1817 apply_container_ordering(action_t *action, pe_working_set_t *data_set)
     /*  */
1818 {
1819     /* VMs are also classified as containers for these purposes... in
1820      * that they both involve a 'thing' running on a real or remote
1821      * cluster node.
1822      *
1823      * This allows us to be smarter about the type and extent of
1824      * recovery actions required in various scenarios
1825      */
1826     resource_t *remote_rsc = NULL;
1827     resource_t *container = NULL;
1828     enum action_tasks task = text2task(action->task);
1829 
1830     CRM_ASSERT(action->rsc);
1831     CRM_ASSERT(action->node);
1832     CRM_ASSERT(is_remote_node(action->node));
1833 
1834     remote_rsc = action->node->details->remote_rsc;
1835     CRM_ASSERT(remote_rsc);
1836 
1837     container = remote_rsc->container;
1838     CRM_ASSERT(container);
1839 
1840     if(is_set(container->flags, pe_rsc_failed)) {
1841         pe_fence_node(data_set, action->node, "container failed");
1842     }
1843 
1844     crm_trace("Order %s action %s relative to %s%s for %s%s",
1845               action->task, action->uuid,
1846               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
1847               remote_rsc->id,
1848               is_set(container->flags, pe_rsc_failed)? "failed " : "",
1849               container->id);
1850 
1851     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
1852         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
1853         /* Migration ops map to "no_action", but we need to apply the same
1854          * ordering as for stop or demote (see get_router_node()).
1855          */
1856         task = stop_rsc;
1857     }
1858 
1859     switch (task) {
1860         case start_rsc:
1861         case action_promote:
1862             /* Force resource recovery if the container is recovered */
1863             order_start_then_action(container, action, pe_order_implies_then,
1864                                     data_set);
1865 
1866             /* Wait for the connection resource to be up too */
1867             order_start_then_action(remote_rsc, action, pe_order_none,
1868                                     data_set);
1869             break;
1870 
1871         case stop_rsc:
1872         case action_demote:
1873             if (is_set(container->flags, pe_rsc_failed)) {
1874                 /* When the container representing a guest node fails, any stop
1875                  * or demote actions for resources running on the guest node
1876                  * are implied by the container stopping. This is similar to
1877                  * how fencing operations work for cluster nodes and remote
1878                  * nodes.
1879                  */
1880             } else {
1881                 /* Ensure the operation happens before the connection is brought
1882                  * down.
1883                  *
1884                  * If we really wanted to, we could order these after the
1885                  * connection start, IFF the container's current role was
1886                  * stopped (otherwise we re-introduce an ordering loop when the
1887                  * connection is restarting).
1888                  */
1889                 order_action_then_stop(action, remote_rsc, pe_order_none,
1890                                        data_set);
1891             }
1892             break;
1893 
1894         default:
1895             /* Wait for the connection resource to be up */
1896             if (is_recurring_action(action)) {
1897                 /* In case we ever get the recovery logic wrong, force
1898                  * recurring monitors to be restarted, even if just
1899                  * the connection was re-established
1900                  */
1901                 if(task != no_action) {
1902                     order_start_then_action(remote_rsc, action,
1903                                             pe_order_implies_then, data_set);
1904                 }
1905             } else {
1906                 order_start_then_action(remote_rsc, action, pe_order_none,
1907                                         data_set);
1908             }
1909             break;
1910     }
1911 }
1912 
1913 static enum remote_connection_state
1914 get_remote_node_state(pe_node_t *node) 
     /*  */
1915 {
1916     resource_t *remote_rsc = NULL;
1917     node_t *cluster_node = NULL;
1918 
1919     CRM_ASSERT(node);
1920 
1921     remote_rsc = node->details->remote_rsc;
1922     CRM_ASSERT(remote_rsc);
1923 
1924     if(remote_rsc->running_on) {
1925         cluster_node = remote_rsc->running_on->data;
1926     }
1927 
1928 
1929     /* If the cluster node the remote connection resource resides on
1930      * is unclean or went offline, we can't process any operations
1931      * on that remote node until after it starts elsewhere.
1932      */
1933     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
1934         /* The connection resource is not going to run anywhere */
1935 
1936         if (cluster_node && cluster_node->details->unclean) {
1937             /* The remote connection is failed because its resource is on a
1938              * failed node and can't be recovered elsewhere, so we must fence.
1939              */
1940             return remote_state_failed;
1941         }
1942 
1943         if (is_not_set(remote_rsc->flags, pe_rsc_failed)) {
1944             /* Connection resource is cleanly stopped */
1945             return remote_state_stopped;
1946         }
1947 
1948         /* Connection resource is failed */
1949 
1950         if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
1951             && remote_rsc->remote_reconnect_interval
1952             && node->details->remote_was_fenced) {
1953 
1954             /* We won't know whether the connection is recoverable until the
1955              * reconnect interval expires and we reattempt connection.
1956              */
1957             return remote_state_unknown;
1958         }
1959 
1960         /* The remote connection is in a failed state. If there are any
1961          * resources known to be active on it (stop) or in an unknown state
1962          * (probe), we must assume the worst and fence it.
1963          */
1964         return remote_state_failed;
1965 
1966     } else if (cluster_node == NULL) {
1967         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
1968         return remote_state_unknown;
1969 
1970     } else if(cluster_node->details->unclean == TRUE
1971               || cluster_node->details->online == FALSE) {
1972         /* Connection is running on a dead node, see if we can recover it first */
1973         return remote_state_resting;
1974 
1975     } else if (g_list_length(remote_rsc->running_on) > 1
1976                && remote_rsc->partial_migration_source
1977                && remote_rsc->partial_migration_target) {
1978         /* We're in the middle of migrating a connection resource,
1979          * wait until after the resource migrates before performing
1980          * any actions.
1981          */
1982         return remote_state_resting;
1983 
1984     }
1985     return remote_state_alive;
1986 }
1987 
1988 static void
1989 apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
     /*  */
1990 {
1991     resource_t *remote_rsc = NULL;
1992     node_t *cluster_node = NULL;
1993     enum action_tasks task = text2task(action->task);
1994     enum remote_connection_state state = get_remote_node_state(action->node);
1995 
1996     enum pe_ordering order_opts = pe_order_none;
1997 
1998     if (action->rsc == NULL) {
1999         return;
2000     }
2001 
2002     CRM_ASSERT(action->node);
2003     CRM_ASSERT(is_remote_node(action->node));
2004 
2005     remote_rsc = action->node->details->remote_rsc;
2006     CRM_ASSERT(remote_rsc);
2007 
2008     if(remote_rsc->running_on) {
2009         cluster_node = remote_rsc->running_on->data;
2010     }
2011 
2012     crm_trace("Order %s action %s relative to %s%s (state %d)",
2013               action->task, action->uuid,
2014               is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
2015               remote_rsc->id, state);
2016 
2017     if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
2018         || safe_str_eq(action->task, CRMD_ACTION_MIGRATE)) {
2019         /* Migration ops map to "no_action", but we need to apply the same
2020          * ordering as for stop or demote (see get_router_node()).
2021          */
2022         task = stop_rsc;
2023     }
2024 
2025     switch (task) {
2026         case start_rsc:
2027         case action_promote:
2028             order_opts = pe_order_none;
2029 
2030             if (state == remote_state_failed) {
2031                 /* Force recovery, by making this action required */
2032                 order_opts |= pe_order_implies_then;
2033             }
2034 
2035             /* Ensure connection is up before running this action */
2036             order_start_then_action(remote_rsc, action, order_opts, data_set);
2037             break;
2038 
2039         case stop_rsc:
2040             /* Handle special case with remote node where stop actions need to be
2041              * ordered after the connection resource starts somewhere else.
2042              */
2043             if(state == remote_state_resting) {
2044                 /* Wait for the connection resource to be up and assume everything is as we left it */
2045                 order_start_then_action(remote_rsc, action, pe_order_none,
2046                                         data_set);
2047 
2048             } else {
2049                 if(state == remote_state_failed) {
2050                     /* We would only be here if the resource is
2051                      * running on the remote node.  Since we have no
2052                      * way to stop it, it is necessary to fence the
2053                      * node.
2054                      */
2055                     pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");
2056                 }
2057 
2058                 order_action_then_stop(action, remote_rsc,
2059                                        pe_order_implies_first, data_set);
2060             }
2061             break;
2062 
2063         case action_demote:
2064             /* Only order this demote relative to the connection start if the
2065              * connection isn't being torn down. Otherwise, the demote would be
2066              * blocked because the connection start would not be allowed.
2067              */
2068             if(state == remote_state_resting || state == remote_state_unknown) {
2069                 order_start_then_action(remote_rsc, action, pe_order_none,
2070                                         data_set);
2071             } /* Otherwise we can rely on the stop ordering */
2072             break;
2073 
2074         default:
2075             /* Wait for the connection resource to be up */
2076             if (is_recurring_action(action)) {
2077                 /* In case we ever get the recovery logic wrong, force
2078                  * recurring monitors to be restarted, even if just
2079                  * the connection was re-established
2080                  */
2081                 order_start_then_action(remote_rsc, action,
2082                                         pe_order_implies_then, data_set);
2083 
2084             } else {
2085                 if(task == monitor_rsc && state == remote_state_failed) {
2086                     /* We would only be here if we do not know the
2087                      * state of the resource on the remote node.
2088                      * Since we have no way to find out, it is
2089                      * necessary to fence the node.
2090                      */
2091                     pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");
2092                 }
2093 
2094                 if(cluster_node && state == remote_state_stopped) {
2095                     /* The connection is currently up, but is going
2096                      * down permanently.
2097                      *
2098                      * Make sure we check services are actually
2099                      * stopped _before_ we let the connection get
2100                      * closed
2101                      */
2102                     order_action_then_stop(action, remote_rsc,
2103                                            pe_order_runnable_left, data_set);
2104 
2105                 } else {
2106                     order_start_then_action(remote_rsc, action, pe_order_none,
2107                                             data_set);
2108                 }
2109             }
2110             break;
2111     }
2112 }
2113 
2114 static void
2115 apply_remote_node_ordering(pe_working_set_t *data_set)
     /*  */
2116 {
2117     if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
2118         return;
2119     }
2120 
2121     for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2122         action_t *action = (action_t *) gIter->data;
2123         resource_t *remote = NULL;
2124 
2125         // We are only interested in resource actions
2126         if (action->rsc == NULL) {
2127             continue;
2128         }
2129 
2130         /* Special case: If we are clearing the failcount of an actual
2131          * remote connection resource, then make sure this happens before
2132          * any start of the resource in this transition.
2133          */
2134         if (action->rsc->is_remote_node &&
2135             safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) {
2136 
2137             custom_action_order(action->rsc,
2138                 NULL,
2139                 action,
2140                 action->rsc,
2141                 generate_op_key(action->rsc->id, RSC_START, 0),
2142                 NULL,
2143                 pe_order_optional,
2144                 data_set);
2145 
2146             continue;
2147         }
2148 
2149         // We are only interested in actions allocated to a node
2150         if (action->node == NULL) {
2151             continue;
2152         }
2153 
2154         if (is_remote_node(action->node) == FALSE) {
2155             continue;
2156         }
2157 
2158         /* We are only interested in real actions.
2159          *
2160          * @TODO This is probably wrong; pseudo-actions might be converted to
2161          * real actions and vice versa later in update_actions() at the end of
2162          * stage7().
2163          */
2164         if (is_set(action->flags, pe_action_pseudo)) {
2165             continue;
2166         }
2167 
2168         remote = action->node->details->remote_rsc;
2169         if (remote == NULL) {
2170             // Orphaned
2171             continue;
2172         }
2173 
2174         /* The action occurs across a remote connection, so create
2175          * ordering constraints that guarantee the action occurs while the node
2176          * is active (after start, before stop ... things like that).
2177          *
2178          * This is somewhat brittle in that we need to make sure the results of
2179          * this ordering are compatible with the result of get_router_node().
2180          * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
2181          * of this logic rather than action2xml().
2182          */
2183         if (remote->container) {
2184             crm_trace("Container ordering for %s", action->uuid);
2185             apply_container_ordering(action, data_set);
2186 
2187         } else {
2188             crm_trace("Remote ordering for %s", action->uuid);
2189             apply_remote_ordering(action, data_set);
2190         }
2191     }
2192 }
2193 
2194 static void
2195 order_probes(pe_working_set_t * data_set) 
     /*  */
2196 {
2197 #if 0
2198     GListPtr gIter = NULL;
2199 
2200     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2201         resource_t *rsc = (resource_t *) gIter->data;
2202 
2203         /* Given "A then B", we would prefer to wait for A to be
2204          * started before probing B.
2205          *
2206          * If A was a filesystem on which the binaries and data for B
2207          * lived, it would have been useful if the author of B's agent
2208          * could assume that A is running before B.monitor will be
2209          * called.
2210          *
2211          * However we can't _only_ probe once A is running, otherwise
2212          * we'd not detect the state of B if A could not be started
2213          * for some reason.
2214          *
2215          * In practice however, we cannot even do an opportunistic
2216          * version of this because B may be moving:
2217          *
2218          *   B.probe -> B.start
2219          *   B.probe -> B.stop
2220          *   B.stop -> B.start
2221          *   A.stop -> A.start
2222          *   A.start -> B.probe
2223          *
2224          * So far so good, but if we add the result of this code:
2225          *
2226          *   B.stop -> A.stop
2227          *
2228          * Then we get a loop:
2229          *
2230          *   B.probe -> B.stop -> A.stop -> A.start -> B.probe
2231          *
2232          * We could kill the 'B.probe -> B.stop' dependency, but that
2233          * could mean stopping B "too" soon, because B.start must wait
2234          * for the probes to complete.
2235          *
2236          * Another option is to allow it only if A is a non-unique
2237          * clone with clone-max == node-max (since we'll never be
2238          * moving it).  However, we could still be stopping one
2239          * instance at the same time as starting another.
2240 
2241          * The complexity of checking for allowed conditions combined
2242          * with the ever narrowing usecase suggests that this code
2243          * should remain disabled until someone gets smarter.
2244          */
2245         action_t *start = NULL;
2246         GListPtr actions = NULL;
2247         GListPtr probes = NULL;
2248         char *key = NULL;
2249 
2250         key = start_key(rsc);
2251         actions = find_actions(rsc->actions, key, NULL);
2252         free(key);
2253 
2254         if (actions) {
2255             start = actions->data;
2256             g_list_free(actions);
2257         }
2258 
2259         if(start == NULL) {
2260             crm_err("No start action for %s", rsc->id);
2261             continue;
2262         }
2263 
2264         key = generate_op_key(rsc->id, CRMD_ACTION_STATUS, 0);
2265         probes = find_actions(rsc->actions, key, NULL);
2266         free(key);
2267 
2268         for (actions = start->actions_before; actions != NULL; actions = actions->next) {
2269             action_wrapper_t *before = (action_wrapper_t *) actions->data;
2270 
2271             GListPtr pIter = NULL;
2272             action_t *first = before->action;
2273             resource_t *first_rsc = first->rsc;
2274 
2275             if(first->required_runnable_before) {
2276                 GListPtr clone_actions = NULL;
2277                 for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
2278                     before = (action_wrapper_t *) clone_actions->data;
2279 
2280                     crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
2281 
2282                     CRM_ASSERT(before->action->rsc);
2283                     first_rsc = before->action->rsc;
2284                     break;
2285                 }
2286 
2287             } else if(safe_str_neq(first->task, RSC_START)) {
2288                 crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
2289             }
2290 
2291             if(first_rsc == NULL) {
2292                 continue;
2293 
2294             } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
2295                 crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
2296                 continue;
2297 
2298             } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
2299                 crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
2300                 continue;
2301             }
2302 
2303             crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
2304 
2305             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
2306                 action_t *probe = (action_t *) pIter->data;
2307 
2308                 crm_err("Ordering %s before %s", first->uuid, probe->uuid);
2309                 order_actions(first, probe, pe_order_optional);
2310             }
2311         }
2312     }
2313 #endif
2314 }
2315 
2316 gboolean
2317 stage7(pe_working_set_t * data_set)
     /*  */
2318 {
2319     GListPtr gIter = NULL;
2320 
2321     crm_trace("Applying ordering constraints");
2322 
2323     /* Don't ask me why, but apparently they need to be processed in
2324      * the order they were created in... go figure
2325      *
2326      * Also g_list_append() has horrendous performance characteristics
2327      * So we need to use g_list_prepend() and then reverse the list here
2328      */
2329     data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
2330 
2331     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
2332         order_constraint_t *order = (order_constraint_t *) gIter->data;
2333         resource_t *rsc = order->lh_rsc;
2334 
2335         crm_trace("Applying ordering constraint: %d", order->id);
2336 
2337         if (rsc != NULL) {
2338             crm_trace("rsc_action-to-*");
2339             rsc_order_first(rsc, order, data_set);
2340             continue;
2341         }
2342 
2343         rsc = order->rh_rsc;
2344         if (rsc != NULL) {
2345             crm_trace("action-to-rsc_action");
2346             rsc_order_then(order->lh_action, rsc, order);
2347 
2348         } else {
2349             crm_trace("action-to-action");
2350             order_actions(order->lh_action, order->rh_action, order->type);
2351         }
2352     }
2353 
2354     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2355         action_t *action = (action_t *) gIter->data;
2356 
2357         update_colo_start_chain(action);
2358     }
2359 
2360     crm_trace("Ordering probes");
2361     order_probes(data_set);
2362 
2363     crm_trace("Updating %d actions", g_list_length(data_set->actions));
2364     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2365         action_t *action = (action_t *) gIter->data;
2366 
2367         update_action(action);
2368     }
2369 
2370     LogNodeActions(data_set, FALSE);
2371     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2372         resource_t *rsc = (resource_t *) gIter->data;
2373 
2374         LogActions(rsc, data_set, FALSE);
2375     }
2376     return TRUE;
2377 }
2378 
2379 int transition_id = -1;
2380 
2381 /*
2382  * Create a dependency graph to send to the transitioner (via the CRMd)
2383  */
2384 gboolean
2385 stage8(pe_working_set_t * data_set)
     /*  */
2386 {
2387     GListPtr gIter = NULL;
2388     const char *value = NULL;
2389 
2390     transition_id++;
2391     crm_trace("Creating transition graph %d.", transition_id);
2392 
2393     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
2394 
2395     value = pe_pref(data_set->config_hash, "cluster-delay");
2396     crm_xml_add(data_set->graph, "cluster-delay", value);
2397 
2398     value = pe_pref(data_set->config_hash, "stonith-timeout");
2399     crm_xml_add(data_set->graph, "stonith-timeout", value);
2400 
2401     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
2402 
2403     if (is_set(data_set->flags, pe_flag_start_failure_fatal)) {
2404         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
2405     } else {
2406         crm_xml_add(data_set->graph, "failed-start-offset", "1");
2407     }
2408 
2409     value = pe_pref(data_set->config_hash, "batch-limit");
2410     crm_xml_add(data_set->graph, "batch-limit", value);
2411 
2412     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
2413 
2414     value = pe_pref(data_set->config_hash, "migration-limit");
2415     if (crm_int_helper(value, NULL) > 0) {
2416         crm_xml_add(data_set->graph, "migration-limit", value);
2417     }
2418 
2419 /* errors...
2420    slist_iter(action, action_t, action_list, lpc,
2421    if(action->optional == FALSE && action->runnable == FALSE) {
2422    print_action("Ignoring", action, TRUE);
2423    }
2424    );
2425 */
2426 
2427     gIter = data_set->resources;
2428     for (; gIter != NULL; gIter = gIter->next) {
2429         resource_t *rsc = (resource_t *) gIter->data;
2430 
2431         pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
2432         rsc->cmds->expand(rsc, data_set);
2433     }
2434 
2435     crm_log_xml_trace(data_set->graph, "created resource-driven action list");
2436 
2437     /* pseudo action to distribute list of nodes with maintenance state update */
2438     add_maintenance_update(data_set);
2439 
2440     /* catch any non-resource specific actions */
2441     crm_trace("processing non-resource actions");
2442 
2443     gIter = data_set->actions;
2444     for (; gIter != NULL; gIter = gIter->next) {
2445         action_t *action = (action_t *) gIter->data;
2446 
2447         if (action->rsc
2448             && action->node
2449             && action->node->details->shutdown
2450             && is_not_set(action->rsc->flags, pe_rsc_maintenance)
2451             && is_not_set(action->flags, pe_action_optional)
2452             && is_not_set(action->flags, pe_action_runnable)
2453             && crm_str_eq(action->task, RSC_STOP, TRUE)
2454             ) {
2455             /* Eventually we should just ignore the 'fence' case
2456              * But for now it's the best way to detect (in CTS) when
2457              * CIB resource updates are being lost
2458              */
2459             if (is_set(data_set->flags, pe_flag_have_quorum)
2460                 || data_set->no_quorum_policy == no_quorum_ignore) {
2461                 crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
2462                          action->node->details->unclean ? "fence" : "shut down",
2463                          action->node->details->uname, action->rsc->id,
2464                          is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked",
2465                          is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "",
2466                          action->uuid);
2467             }
2468         }
2469 
2470         graph_element_from_action(action, data_set);
2471     }
2472 
2473     crm_log_xml_trace(data_set->graph, "created generic action list");
2474     crm_trace("Created transition graph %d.", transition_id);
2475 
2476     return TRUE;
2477 }
2478 
2479 void
2480 LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
     /*  */
2481 {
2482     GListPtr gIter = NULL;
2483 
2484     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2485         char *node_name = NULL;
2486         char *task = NULL;
2487         action_t *action = (action_t *) gIter->data;
2488 
2489         if (action->rsc != NULL) {
2490             continue;
2491         } else if (is_set(action->flags, pe_action_optional)) {
2492             continue;
2493         }
2494 
2495         if (is_container_remote_node(action->node)) {
2496             node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
2497         } else if(action->node) {
2498             node_name = crm_strdup_printf("%s", action->node->details->uname);
2499         }
2500 
2501 
2502         if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
2503             task = strdup("Shutdown");
2504         } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
2505             const char *op = g_hash_table_lookup(action->meta, "stonith_action");
2506             task = crm_strdup_printf("Fence (%s)", op);
2507         }
2508 
2509         if(task == NULL) {
2510             /* Nothing to report */
2511         } else if(terminal && action->reason) {
2512             printf(" * %s %s '%s'\n", task, node_name, action->reason);
2513         } else if(terminal) {
2514             printf(" * %s %s\n", task, node_name);
2515         } else if(action->reason) {
2516             crm_notice(" * %s %s '%s'\n", task, node_name, action->reason);
2517         } else {
2518             crm_notice(" * %s %s\n", task, node_name);
2519         }
2520 
2521         free(node_name);
2522         free(task);
2523     }
2524 }
2525 
2526 void
2527 cleanup_alloc_calculations(pe_working_set_t * data_set)
     /*  */
2528 {
2529     if (data_set == NULL) {
2530         return;
2531     }
2532 
2533     crm_trace("deleting %d order cons: %p",
2534               g_list_length(data_set->ordering_constraints), data_set->ordering_constraints);
2535     pe_free_ordering(data_set->ordering_constraints);
2536     data_set->ordering_constraints = NULL;
2537 
2538     crm_trace("deleting %d node cons: %p",
2539               g_list_length(data_set->placement_constraints), data_set->placement_constraints);
2540     pe_free_rsc_to_node(data_set->placement_constraints);
2541     data_set->placement_constraints = NULL;
2542 
2543     crm_trace("deleting %d inter-resource cons: %p",
2544               g_list_length(data_set->colocation_constraints), data_set->colocation_constraints);
2545     g_list_free_full(data_set->colocation_constraints, free);
2546     data_set->colocation_constraints = NULL;
2547 
2548     crm_trace("deleting %d ticket deps: %p",
2549               g_list_length(data_set->ticket_constraints), data_set->ticket_constraints);
2550     g_list_free_full(data_set->ticket_constraints, free);
2551     data_set->ticket_constraints = NULL;
2552 
2553     cleanup_calculations(data_set);
2554 }