lib/pacemaker/pcmk_sched

/* */
This source file includes following definitions.
state2text
check_rsc_parameters
CancelXmlOp
check_action_definition
check_params
check_actions_for
find_rsc_list
check_actions
apply_placement_constraints
failcount_clear_action_exists
check_migration_threshold
common_apply_stickiness
complex_set_cmds
set_alloc_actions
calculate_system_health
apply_system_health
stage0
probe_resources
rsc_discover_filter
shutdown_time
apply_shutdown_lock
stage2
stage3
stage4
convert_const_pointer
sort_rsc_process_order
allocate_resources
order_start_then_action
order_action_then_stop
cleanup_orphans
stage5
is_managed
any_managed_resources
fence_guest
stage6
find_actions_by_task
rsc_order_then
rsc_order_first
is_recurring_action
apply_container_ordering
get_remote_node_state
apply_remote_ordering
apply_remote_node_ordering
order_first_probe_unneeded
order_first_probes_imply_stops
order_first_probe_then_restart_repromote
clear_actions_tracking_flag
order_first_rsc_probes
order_first_probes
order_then_probes
order_probes
stage7
pcmk__log_transition_summary
stage8
LogNodeActions
   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/cib.h>
  16 #include <crm/msg_xml.h>
  17 #include <crm/common/xml.h>
  18 #include <crm/common/xml_internal.h>
  19 
  20 #include <glib.h>
  21 
  22 #include <crm/pengine/status.h>
  23 #include <pacemaker-internal.h>
  24 
  25 CRM_TRACE_INIT_DATA(pacemaker);
  26 
  27 extern bool pcmk__is_daemon;
  28 
  29 void set_alloc_actions(pe_working_set_t * data_set);
  30 extern void ReloadRsc(pe_resource_t * rsc, pe_node_t *node, pe_working_set_t * data_set);
  31 extern gboolean DeleteRsc(pe_resource_t * rsc, pe_node_t * node, gboolean optional, pe_working_set_t * data_set);
  32 static void apply_remote_node_ordering(pe_working_set_t *data_set);
  33 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
  34 
  35 enum remote_connection_state {
  36     remote_state_unknown = 0,
  37     remote_state_alive = 1,
  38     remote_state_resting = 2,
  39     remote_state_failed = 3,
  40     remote_state_stopped = 4
  41 };
  42 
  43 static const char *
  44 state2text(enum remote_connection_state state)
     /*  */
  45 {
  46     switch (state) {
  47         case remote_state_unknown:
  48             return "unknown";
  49         case remote_state_alive:
  50             return "alive";
  51         case remote_state_resting:
  52             return "resting";
  53         case remote_state_failed:
  54             return "failed";
  55         case remote_state_stopped:
  56             return "stopped";
  57     }
  58 
  59     return "impossible";
  60 }
  61 
  62 resource_alloc_functions_t resource_class_alloc_functions[] = {
  63     {
  64      pcmk__native_merge_weights,
  65      pcmk__native_allocate,
  66      native_create_actions,
  67      native_create_probe,
  68      native_internal_constraints,
  69      native_rsc_colocation_lh,
  70      native_rsc_colocation_rh,
  71      native_rsc_location,
  72      native_action_flags,
  73      native_update_actions,
  74      native_expand,
  75      native_append_meta,
  76      },
  77     {
  78      pcmk__group_merge_weights,
  79      pcmk__group_allocate,
  80      group_create_actions,
  81      native_create_probe,
  82      group_internal_constraints,
  83      group_rsc_colocation_lh,
  84      group_rsc_colocation_rh,
  85      group_rsc_location,
  86      group_action_flags,
  87      group_update_actions,
  88      group_expand,
  89      group_append_meta,
  90      },
  91     {
  92      pcmk__native_merge_weights,
  93      pcmk__clone_allocate,
  94      clone_create_actions,
  95      clone_create_probe,
  96      clone_internal_constraints,
  97      clone_rsc_colocation_lh,
  98      clone_rsc_colocation_rh,
  99      clone_rsc_location,
 100      clone_action_flags,
 101      pcmk__multi_update_actions,
 102      clone_expand,
 103      clone_append_meta,
 104      },
 105     {
 106      pcmk__native_merge_weights,
 107      pcmk__bundle_allocate,
 108      pcmk__bundle_create_actions,
 109      pcmk__bundle_create_probe,
 110      pcmk__bundle_internal_constraints,
 111      pcmk__bundle_rsc_colocation_lh,
 112      pcmk__bundle_rsc_colocation_rh,
 113      pcmk__bundle_rsc_location,
 114      pcmk__bundle_action_flags,
 115      pcmk__multi_update_actions,
 116      pcmk__bundle_expand,
 117      pcmk__bundle_append_meta,
 118      }
 119 };
 120 
 121 static gboolean
 122 check_rsc_parameters(pe_resource_t * rsc, pe_node_t * node, xmlNode * rsc_entry,
     /*  */
 123                      gboolean active_here, pe_working_set_t * data_set)
 124 {
 125     int attr_lpc = 0;
 126     gboolean force_restart = FALSE;
 127     gboolean delete_resource = FALSE;
 128     gboolean changed = FALSE;
 129 
 130     const char *value = NULL;
 131     const char *old_value = NULL;
 132 
 133     const char *attr_list[] = {
 134         XML_ATTR_TYPE,
 135         XML_AGENT_ATTR_CLASS,
 136         XML_AGENT_ATTR_PROVIDER
 137     };
 138 
 139     for (; attr_lpc < PCMK__NELEM(attr_list); attr_lpc++) {
 140         value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
 141         old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
 142         if (value == old_value  /* i.e. NULL */
 143             || pcmk__str_eq(value, old_value, pcmk__str_none)) {
 144             continue;
 145         }
 146 
 147         changed = TRUE;
 148         trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
 149         if (active_here) {
 150             force_restart = TRUE;
 151             crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
 152                        rsc->id, node->details->uname, attr_list[attr_lpc],
 153                        crm_str(old_value), crm_str(value));
 154         }
 155     }
 156     if (force_restart) {
 157         /* make sure the restart happens */
 158         stop_action(rsc, node, FALSE);
 159         pe__set_resource_flags(rsc, pe_rsc_start_pending);
 160         delete_resource = TRUE;
 161 
 162     } else if (changed) {
 163         delete_resource = TRUE;
 164     }
 165     return delete_resource;
 166 }
 167 
 168 static void
 169 CancelXmlOp(pe_resource_t * rsc, xmlNode * xml_op, pe_node_t * active_node,
     /*  */
 170             const char *reason, pe_working_set_t * data_set)
 171 {
 172     guint interval_ms = 0;
 173     pe_action_t *cancel = NULL;
 174 
 175     const char *task = NULL;
 176     const char *call_id = NULL;
 177 
 178     CRM_CHECK(xml_op != NULL, return);
 179     CRM_CHECK(active_node != NULL, return);
 180 
 181     task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 182     call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
 183     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 184 
 185     crm_info("Action " PCMK__OP_FMT " on %s will be stopped: %s",
 186              rsc->id, task, interval_ms,
 187              active_node->details->uname, (reason? reason : "unknown"));
 188 
 189     cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
 190     add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
 191     custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
 192 }
 193 
 194 static gboolean
 195 check_action_definition(pe_resource_t * rsc, pe_node_t * active_node, xmlNode * xml_op,
     /*  */
 196                         pe_working_set_t * data_set)
 197 {
 198     char *key = NULL;
 199     guint interval_ms = 0;
 200     const op_digest_cache_t *digest_data = NULL;
 201     gboolean did_change = FALSE;
 202 
 203     const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
 204     const char *digest_secure = NULL;
 205 
 206     CRM_CHECK(active_node != NULL, return FALSE);
 207 
 208     crm_element_value_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 209     if (interval_ms > 0) {
 210         xmlNode *op_match = NULL;
 211 
 212         /* we need to reconstruct the key because of the way we used to construct resource IDs */
 213         key = pcmk__op_key(rsc->id, task, interval_ms);
 214 
 215         pe_rsc_trace(rsc, "Checking parameters for %s", key);
 216         op_match = find_rsc_op_entry(rsc, key);
 217 
 218         if ((op_match == NULL)
 219             && pcmk_is_set(data_set->flags, pe_flag_stop_action_orphans)) {
 220             CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
 221             free(key);
 222             return TRUE;
 223 
 224         } else if (op_match == NULL) {
 225             pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
 226             free(key);
 227             return TRUE;
 228         }
 229         free(key);
 230         key = NULL;
 231     }
 232 
 233     crm_trace("Testing " PCMK__OP_FMT " on %s",
 234               rsc->id, task, interval_ms, active_node->details->uname);
 235     if ((interval_ms == 0) && pcmk__str_eq(task, RSC_STATUS, pcmk__str_casei)) {
 236         /* Reload based on the start action not a probe */
 237         task = RSC_START;
 238 
 239     } else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_MIGRATED, pcmk__str_casei)) {
 240         /* Reload based on the start action not a migrate */
 241         task = RSC_START;
 242     } else if ((interval_ms == 0) && pcmk__str_eq(task, RSC_PROMOTE, pcmk__str_casei)) {
 243         /* Reload based on the start action not a promote */
 244         task = RSC_START;
 245     }
 246 
 247     digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
 248 
 249     if (pcmk_is_set(data_set->flags, pe_flag_sanitized)) {
 250         digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
 251     }
 252 
 253     if(digest_data->rc != RSC_DIGEST_MATCH
 254        && digest_secure
 255        && digest_data->digest_secure_calc
 256        && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
 257         if (!pcmk__is_daemon && data_set->priv != NULL) {
 258             pcmk__output_t *out = data_set->priv;
 259             out->info(out, "Only 'private' parameters to "
 260                       PCMK__OP_FMT " on %s changed: %s", rsc->id, task,
 261                       interval_ms, active_node->details->uname,
 262                       crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC));
 263         }
 264 
 265     } else if (digest_data->rc == RSC_DIGEST_RESTART) {
 266         /* Changes that force a restart */
 267         pe_action_t *required = NULL;
 268 
 269         did_change = TRUE;
 270         key = pcmk__op_key(rsc->id, task, interval_ms);
 271         crm_log_xml_info(digest_data->params_restart, "params:restart");
 272         required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
 273         pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
 274                                   "resource definition change", pe_action_optional, TRUE);
 275 
 276         trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
 277 
 278     } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
 279         // Changes that can potentially be handled by an agent reload
 280         const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
 281 
 282         did_change = TRUE;
 283         trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
 284         crm_log_xml_info(digest_data->params_all, "params:reload");
 285         key = pcmk__op_key(rsc->id, task, interval_ms);
 286 
 287         if (interval_ms > 0) {
 288             pe_action_t *op = NULL;
 289 
 290 #if 0
 291             /* Always reload/restart the entire resource */
 292             ReloadRsc(rsc, active_node, data_set);
 293 #else
 294             /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
 295             op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
 296             pe__set_action_flags(op, pe_action_reschedule);
 297 #endif
 298 
 299         } else if (digest_restart) {
 300             pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
 301 
 302             /* Reload this resource */
 303             ReloadRsc(rsc, active_node, data_set);
 304             free(key);
 305 
 306         } else {
 307             pe_action_t *required = NULL;
 308             pe_rsc_trace(rsc, "Resource %s doesn't support agent reloads",
 309                          rsc->id);
 310 
 311             /* Re-send the start/demote/promote op
 312              * Recurring ops will be detected independently
 313              */
 314             required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
 315             pe_action_set_flag_reason(__func__, __LINE__, required, NULL,
 316                                       "resource definition change", pe_action_optional, TRUE);
 317         }
 318     }
 319 
 320     return did_change;
 321 }
 322 
 323 /*!
 324  * \internal
 325  * \brief Do deferred action checks after allocation
 326  *
 327  * \param[in] data_set  Working set for cluster
 328  */
 329 static void
 330 check_params(pe_resource_t *rsc, pe_node_t *node, xmlNode *rsc_op,
     /*  */
 331              enum pe_check_parameters check, pe_working_set_t *data_set)
 332 {
 333     const char *reason = NULL;
 334     op_digest_cache_t *digest_data = NULL;
 335 
 336     switch (check) {
 337         case pe_check_active:
 338             if (check_action_definition(rsc, node, rsc_op, data_set)
 339                 && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
 340                                     data_set)) {
 341 
 342                 reason = "action definition changed";
 343             }
 344             break;
 345 
 346         case pe_check_last_failure:
 347             digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
 348             switch (digest_data->rc) {
 349                 case RSC_DIGEST_UNKNOWN:
 350                     crm_trace("Resource %s history entry %s on %s has no digest to compare",
 351                               rsc->id, ID(rsc_op), node->details->id);
 352                     break;
 353                 case RSC_DIGEST_MATCH:
 354                     break;
 355                 default:
 356                     reason = "resource parameters have changed";
 357                     break;
 358             }
 359             break;
 360     }
 361 
 362     if (reason) {
 363         pe__clear_failcount(rsc, node, reason, data_set);
 364     }
 365 }
 366 
 367 static void
 368 check_actions_for(xmlNode * rsc_entry, pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
     /*  */
 369 {
 370     GList *gIter = NULL;
 371     int offset = -1;
 372     int stop_index = 0;
 373     int start_index = 0;
 374 
 375     const char *task = NULL;
 376 
 377     xmlNode *rsc_op = NULL;
 378     GList *op_list = NULL;
 379     GList *sorted_op_list = NULL;
 380 
 381     CRM_CHECK(node != NULL, return);
 382 
 383     if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
 384         pe_resource_t *parent = uber_parent(rsc);
 385         if(parent == NULL
 386            || pe_rsc_is_clone(parent) == FALSE
 387            || pcmk_is_set(parent->flags, pe_rsc_unique)) {
 388             pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
 389             DeleteRsc(rsc, node, FALSE, data_set);
 390         } else {
 391             pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
 392         }
 393         return;
 394 
 395     } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
 396         if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
 397             DeleteRsc(rsc, node, FALSE, data_set);
 398         }
 399         pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
 400                      rsc->id, node->details->uname);
 401         return;
 402     }
 403 
 404     pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
 405 
 406     if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
 407         DeleteRsc(rsc, node, FALSE, data_set);
 408     }
 409 
 410     for (rsc_op = pcmk__xe_first_child(rsc_entry); rsc_op != NULL;
 411          rsc_op = pcmk__xe_next(rsc_op)) {
 412 
 413         if (pcmk__str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, pcmk__str_none)) {
 414             op_list = g_list_prepend(op_list, rsc_op);
 415         }
 416     }
 417 
 418     sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
 419     calculate_active_ops(sorted_op_list, &start_index, &stop_index);
 420 
 421     for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
 422         xmlNode *rsc_op = (xmlNode *) gIter->data;
 423         guint interval_ms = 0;
 424 
 425         offset++;
 426 
 427         if (start_index < stop_index) {
 428             /* stopped */
 429             continue;
 430         } else if (offset < start_index) {
 431             /* action occurred prior to a start */
 432             continue;
 433         }
 434 
 435         task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
 436         crm_element_value_ms(rsc_op, XML_LRM_ATTR_INTERVAL_MS, &interval_ms);
 437 
 438         if ((interval_ms > 0) &&
 439             (pcmk_is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
 440             // Maintenance mode cancels recurring operations
 441             CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
 442 
 443         } else if ((interval_ms > 0) || pcmk__strcase_any_of(task, RSC_STATUS, RSC_START,
 444                                                              RSC_PROMOTE, RSC_MIGRATED, NULL)) {
 445             /* If a resource operation failed, and the operation's definition
 446              * has changed, clear any fail count so they can be retried fresh.
 447              */
 448 
 449             if (pe__bundle_needs_remote_name(rsc, data_set)) {
 450                 /* We haven't allocated resources to nodes yet, so if the
 451                  * REMOTE_CONTAINER_HACK is used, we may calculate the digest
 452                  * based on the literal "#uname" value rather than the properly
 453                  * substituted value. That would mistakenly make the action
 454                  * definition appear to have been changed. Defer the check until
 455                  * later in this case.
 456                  */
 457                 pe__add_param_check(rsc_op, rsc, node, pe_check_active,
 458                                     data_set);
 459 
 460             } else if (check_action_definition(rsc, node, rsc_op, data_set)
 461                 && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
 462                                     data_set)) {
 463                 pe__clear_failcount(rsc, node, "action definition changed",
 464                                     data_set);
 465             }
 466         }
 467     }
 468     g_list_free(sorted_op_list);
 469 }
 470 
 471 static GList *
 472 find_rsc_list(GList *result, pe_resource_t * rsc, const char *id, gboolean renamed_clones,
     /*  */
 473               gboolean partial, pe_working_set_t * data_set)
 474 {
 475     GList *gIter = NULL;
 476     gboolean match = FALSE;
 477 
 478     if (id == NULL) {
 479         return NULL;
 480     }
 481 
 482     if (rsc == NULL) {
 483         if (data_set == NULL) {
 484             return NULL;
 485         }
 486         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
 487             pe_resource_t *child = (pe_resource_t *) gIter->data;
 488 
 489             result = find_rsc_list(result, child, id, renamed_clones, partial,
 490                                    NULL);
 491         }
 492         return result;
 493     }
 494 
 495     if (partial) {
 496         if (strstr(rsc->id, id)) {
 497             match = TRUE;
 498 
 499         } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
 500             match = TRUE;
 501         }
 502 
 503     } else {
 504         if (strcmp(rsc->id, id) == 0) {
 505             match = TRUE;
 506 
 507         } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
 508             match = TRUE;
 509         }
 510     }
 511 
 512     if (match) {
 513         result = g_list_prepend(result, rsc);
 514     }
 515 
 516     if (rsc->children) {
 517         gIter = rsc->children;
 518         for (; gIter != NULL; gIter = gIter->next) {
 519             pe_resource_t *child = (pe_resource_t *) gIter->data;
 520 
 521             result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
 522         }
 523     }
 524 
 525     return result;
 526 }
 527 
 528 static void
 529 check_actions(pe_working_set_t * data_set)
     /*  */
 530 {
 531     const char *id = NULL;
 532     pe_node_t *node = NULL;
 533     xmlNode *lrm_rscs = NULL;
 534     xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
 535 
 536     xmlNode *node_state = NULL;
 537 
 538     for (node_state = pcmk__xe_first_child(status); node_state != NULL;
 539          node_state = pcmk__xe_next(node_state)) {
 540 
 541         if (pcmk__str_eq((const char *)node_state->name, XML_CIB_TAG_STATE,
 542                          pcmk__str_none)) {
 543             id = crm_element_value(node_state, XML_ATTR_ID);
 544             lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
 545             lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
 546 
 547             node = pe_find_node_id(data_set->nodes, id);
 548 
 549             if (node == NULL) {
 550                 continue;
 551 
 552             /* Still need to check actions for a maintenance node to cancel existing monitor operations */
 553             } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
 554                 crm_trace("Skipping param check for %s: can't run resources",
 555                           node->details->uname);
 556                 continue;
 557             }
 558 
 559             crm_trace("Processing node %s", node->details->uname);
 560             if (node->details->online
 561                 || pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
 562                 xmlNode *rsc_entry = NULL;
 563 
 564                 for (rsc_entry = pcmk__xe_first_child(lrm_rscs);
 565                      rsc_entry != NULL;
 566                      rsc_entry = pcmk__xe_next(rsc_entry)) {
 567 
 568                     if (pcmk__str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, pcmk__str_none)) {
 569 
 570                         if (xml_has_children(rsc_entry)) {
 571                             GList *gIter = NULL;
 572                             GList *result = NULL;
 573                             const char *rsc_id = ID(rsc_entry);
 574 
 575                             CRM_CHECK(rsc_id != NULL, return);
 576 
 577                             result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
 578                             for (gIter = result; gIter != NULL; gIter = gIter->next) {
 579                                 pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 580 
 581                                 if (rsc->variant != pe_native) {
 582                                     continue;
 583                                 }
 584                                 check_actions_for(rsc_entry, rsc, node, data_set);
 585                             }
 586                             g_list_free(result);
 587                         }
 588                     }
 589                 }
 590             }
 591         }
 592     }
 593 }
 594 
 595 static void
 596 apply_placement_constraints(pe_working_set_t * data_set)
     /*  */
 597 {
 598     for (GList *gIter = data_set->placement_constraints;
 599          gIter != NULL; gIter = gIter->next) {
 600         pe__location_t *cons = gIter->data;
 601 
 602         cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
 603     }
 604 }
 605 
 606 static gboolean
 607 failcount_clear_action_exists(pe_node_t * node, pe_resource_t * rsc)
     /*  */
 608 {
 609     gboolean rc = FALSE;
 610     GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);
 611 
 612     if (list) {
 613         rc = TRUE;
 614     }
 615     g_list_free(list);
 616     return rc;
 617 }
 618 
 619 /*!
 620  * \internal
 621  * \brief Force resource away if failures hit migration threshold
 622  *
 623  * \param[in,out] rsc       Resource to check for failures
 624  * \param[in,out] node      Node to check for failures
 625  * \param[in,out] data_set  Cluster working set to update
 626  */
 627 static void
 628 check_migration_threshold(pe_resource_t *rsc, pe_node_t *node,
     /*  */
 629                           pe_working_set_t *data_set)
 630 {
 631     int fail_count, countdown;
 632     pe_resource_t *failed;
 633 
 634     /* Migration threshold of 0 means never force away */
 635     if (rsc->migration_threshold == 0) {
 636         return;
 637     }
 638 
 639     // If we're ignoring failures, also ignore the migration threshold
 640     if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
 641         return;
 642     }
 643 
 644     /* If there are no failures, there's no need to force away */
 645     fail_count = pe_get_failcount(node, rsc, NULL,
 646                                   pe_fc_effective|pe_fc_fillers, NULL,
 647                                   data_set);
 648     if (fail_count <= 0) {
 649         return;
 650     }
 651 
 652     /* How many more times recovery will be tried on this node */
 653     countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
 654 
 655     /* If failed resource has a parent, we'll force the parent away */
 656     failed = rsc;
 657     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
 658         failed = uber_parent(rsc);
 659     }
 660 
 661     if (countdown == 0) {
 662         resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
 663         crm_warn("Forcing %s away from %s after %d failures (max=%d)",
 664                  failed->id, node->details->uname, fail_count,
 665                  rsc->migration_threshold);
 666     } else {
 667         crm_info("%s can fail %d more times on %s before being forced off",
 668                  failed->id, countdown, node->details->uname);
 669     }
 670 }
 671 
 672 static void
 673 common_apply_stickiness(pe_resource_t * rsc, pe_node_t * node, pe_working_set_t * data_set)
     /*  */
 674 {
 675     if (rsc->children) {
 676         GList *gIter = rsc->children;
 677 
 678         for (; gIter != NULL; gIter = gIter->next) {
 679             pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 680 
 681             common_apply_stickiness(child_rsc, node, data_set);
 682         }
 683         return;
 684     }
 685 
 686     if (pcmk_is_set(rsc->flags, pe_rsc_managed)
 687         && rsc->stickiness != 0 && pcmk__list_of_1(rsc->running_on)) {
 688         pe_node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
 689         pe_node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 690 
 691         if (current == NULL) {
 692 
 693         } else if ((match != NULL)
 694                    || pcmk_is_set(data_set->flags, pe_flag_symmetric_cluster)) {
 695             pe_resource_t *sticky_rsc = rsc;
 696 
 697             resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
 698             pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
 699                          " (node=%s, weight=%d)", sticky_rsc->id,
 700                          node->details->uname, rsc->stickiness);
 701         } else {
 702             GHashTableIter iter;
 703             pe_node_t *nIter = NULL;
 704 
 705             pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
 706                          " and node %s is not explicitly allowed", rsc->id, node->details->uname);
 707             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
 708             while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
 709                 crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
 710             }
 711         }
 712     }
 713 
 714     /* Check the migration threshold only if a failcount clear action
 715      * has not already been placed for this resource on the node.
 716      * There is no sense in potentially forcing the resource from this
 717      * node if the failcount is being reset anyway.
 718      *
 719      * @TODO A clear_failcount operation can be scheduled in stage4() via
 720      * check_actions_for(), or in stage5() via check_params(). This runs in
 721      * stage2(), so it cannot detect those, meaning we might check the migration
 722      * threshold when we shouldn't -- worst case, we stop or move the resource,
 723      * then move it back next transition.
 724      */
 725     if (failcount_clear_action_exists(node, rsc) == FALSE) {
 726         check_migration_threshold(rsc, node, data_set);
 727     }
 728 }
 729 
 730 void
 731 complex_set_cmds(pe_resource_t * rsc)
     /*  */
 732 {
 733     GList *gIter = rsc->children;
 734 
 735     rsc->cmds = &resource_class_alloc_functions[rsc->variant];
 736 
 737     for (; gIter != NULL; gIter = gIter->next) {
 738         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 739 
 740         complex_set_cmds(child_rsc);
 741     }
 742 }
 743 
 744 void
 745 set_alloc_actions(pe_working_set_t * data_set)
     /*  */
 746 {
 747 
 748     GList *gIter = data_set->resources;
 749 
 750     for (; gIter != NULL; gIter = gIter->next) {
 751         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
 752 
 753         complex_set_cmds(rsc);
 754     }
 755 }
 756 
 757 static void
 758 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
     /*  */
 759 {
 760     const char *key = (const char *)gKey;
 761     const char *value = (const char *)gValue;
 762     int *system_health = (int *)user_data;
 763 
 764     if (!gKey || !gValue || !user_data) {
 765         return;
 766     }
 767 
 768     if (pcmk__starts_with(key, "#health")) {
 769         int score;
 770 
 771         /* Convert the value into an integer */
 772         score = char2score(value);
 773 
 774         /* Add it to the running total */
 775         *system_health = pe__add_scores(score, *system_health);
 776     }
 777 }
 778 
 779 static gboolean
 780 apply_system_health(pe_working_set_t * data_set)
     /*  */
 781 {
 782     GList *gIter = NULL;
 783     const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
 784     int base_health = 0;
 785 
 786     if (pcmk__str_eq(health_strategy, "none", pcmk__str_null_matches | pcmk__str_casei)) {
 787         /* Prevent any accidental health -> score translation */
 788         pcmk__score_red = 0;
 789         pcmk__score_yellow = 0;
 790         pcmk__score_green = 0;
 791         return TRUE;
 792 
 793     } else if (pcmk__str_eq(health_strategy, "migrate-on-red", pcmk__str_casei)) {
 794 
 795         /* Resources on nodes which have health values of red are
 796          * weighted away from that node.
 797          */
 798         pcmk__score_red = -INFINITY;
 799         pcmk__score_yellow = 0;
 800         pcmk__score_green = 0;
 801 
 802     } else if (pcmk__str_eq(health_strategy, "only-green", pcmk__str_casei)) {
 803 
 804         /* Resources on nodes which have health values of red or yellow
 805          * are forced away from that node.
 806          */
 807         pcmk__score_red = -INFINITY;
 808         pcmk__score_yellow = -INFINITY;
 809         pcmk__score_green = 0;
 810 
 811     } else if (pcmk__str_eq(health_strategy, "progressive", pcmk__str_casei)) {
 812         /* Same as the above, but use the r/y/g scores provided by the user
 813          * Defaults are provided by the pe_prefs table
 814          * Also, custom health "base score" can be used
 815          */
 816         base_health = char2score(pe_pref(data_set->config_hash,
 817                                          "node-health-base"));
 818 
 819     } else if (pcmk__str_eq(health_strategy, "custom", pcmk__str_casei)) {
 820 
 821         /* Requires the admin to configure the rsc_location constaints for
 822          * processing the stored health scores
 823          */
 824         /* TODO: Check for the existence of appropriate node health constraints */
 825         return TRUE;
 826 
 827     } else {
 828         crm_err("Unknown node health strategy: %s", health_strategy);
 829         return FALSE;
 830     }
 831 
 832     crm_info("Applying automated node health strategy: %s", health_strategy);
 833 
 834     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
 835         int system_health = base_health;
 836         pe_node_t *node = (pe_node_t *) gIter->data;
 837 
 838         /* Search through the node hash table for system health entries. */
 839         g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
 840 
 841         crm_info(" Node %s has an combined system health of %d",
 842                  node->details->uname, system_health);
 843 
 844         /* If the health is non-zero, then create a new rsc2node so that the
 845          * weight will be added later on.
 846          */
 847         if (system_health != 0) {
 848 
 849             GList *gIter2 = data_set->resources;
 850 
 851             for (; gIter2 != NULL; gIter2 = gIter2->next) {
 852                 pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
 853 
 854                 rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
 855             }
 856         }
 857     }
 858 
 859     return TRUE;
 860 }
 861 
 862 gboolean
 863 stage0(pe_working_set_t * data_set)
     /*  */
 864 {
 865     xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
 866 
 867     if (data_set->input == NULL) {
 868         return FALSE;
 869     }
 870 
 871     if (!pcmk_is_set(data_set->flags, pe_flag_have_status)) {
 872         crm_trace("Calculating status");
 873         cluster_status(data_set);
 874     }
 875 
 876     set_alloc_actions(data_set);
 877     apply_system_health(data_set);
 878     unpack_constraints(cib_constraints, data_set);
 879 
 880     return TRUE;
 881 }
 882 
 883 /*
 884  * Check nodes for resources started outside of the LRM
 885  */
 886 gboolean
 887 probe_resources(pe_working_set_t * data_set)
     /*  */
 888 {
 889     pe_action_t *probe_node_complete = NULL;
 890 
 891     for (GList *gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
 892         pe_node_t *node = (pe_node_t *) gIter->data;
 893         const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
 894 
 895         if (node->details->online == FALSE) {
 896 
 897             if (pe__is_remote_node(node) && node->details->remote_rsc
 898                 && (get_remote_node_state(node) == remote_state_failed)) {
 899 
 900                 pe_fence_node(data_set, node, "the connection is unrecoverable", FALSE);
 901             }
 902             continue;
 903 
 904         } else if (node->details->unclean) {
 905             continue;
 906 
 907         } else if (node->details->rsc_discovery_enabled == FALSE) {
 908             /* resource discovery is disabled for this node */
 909             continue;
 910         }
 911 
 912         if (probed != NULL && crm_is_true(probed) == FALSE) {
 913             pe_action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
 914                                                   CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
 915 
 916             add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 917             continue;
 918         }
 919 
 920         for (GList *gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
 921             pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
 922 
 923             rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
 924         }
 925     }
 926     return TRUE;
 927 }
 928 
 929 static void
 930 rsc_discover_filter(pe_resource_t *rsc, pe_node_t *node)
     /*  */
 931 {
 932     GList *gIter = rsc->children;
 933     pe_resource_t *top = uber_parent(rsc);
 934     pe_node_t *match;
 935 
 936     if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
 937         return;
 938     }
 939 
 940     for (; gIter != NULL; gIter = gIter->next) {
 941         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
 942         rsc_discover_filter(child_rsc, node);
 943     }
 944 
 945     match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
 946     if (match && match->rsc_discover_mode != pe_discover_exclusive) {
 947         match->weight = -INFINITY;
 948     }
 949 }
 950 
 951 static time_t
 952 shutdown_time(pe_node_t *node, pe_working_set_t *data_set)
     /*  */
 953 {
 954     const char *shutdown = pe_node_attribute_raw(node, XML_CIB_ATTR_SHUTDOWN);
 955     time_t result = 0;
 956 
 957     if (shutdown) {
 958         long long result_ll;
 959 
 960         if (pcmk__scan_ll(shutdown, &result_ll, 0LL) == pcmk_rc_ok) {
 961             result = (time_t) result_ll;
 962         }
 963     }
 964     return result? result : get_effective_time(data_set);
 965 }
 966 
 967 static void
 968 apply_shutdown_lock(pe_resource_t *rsc, pe_working_set_t *data_set)
     /*  */
 969 {
 970     const char *class;
 971 
 972     // Only primitives and (uncloned) groups may be locked
 973     if (rsc->variant == pe_group) {
 974         for (GList *item = rsc->children; item != NULL;
 975              item = item->next) {
 976             apply_shutdown_lock((pe_resource_t *) item->data, data_set);
 977         }
 978     } else if (rsc->variant != pe_native) {
 979         return;
 980     }
 981 
 982     // Fence devices and remote connections can't be locked
 983     class = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 984     if (pcmk__str_eq(class, PCMK_RESOURCE_CLASS_STONITH, pcmk__str_null_matches)
 985         || pe__resource_is_remote_conn(rsc, data_set)) {
 986         return;
 987     }
 988 
 989     if (rsc->lock_node != NULL) {
 990         // The lock was obtained from resource history
 991 
 992         if (rsc->running_on != NULL) {
 993             /* The resource was started elsewhere even though it is now
 994              * considered locked. This shouldn't be possible, but as a
 995              * failsafe, we don't want to disturb the resource now.
 996              */
 997             pe_rsc_info(rsc,
 998                         "Cancelling shutdown lock because %s is already active",
 999                         rsc->id);
1000             pe__clear_resource_history(rsc, rsc->lock_node, data_set);
1001             rsc->lock_node = NULL;
1002             rsc->lock_time = 0;
1003         }
1004 
1005     // Only a resource active on exactly one node can be locked
1006     } else if (pcmk__list_of_1(rsc->running_on)) {
1007         pe_node_t *node = rsc->running_on->data;
1008 
1009         if (node->details->shutdown) {
1010             if (node->details->unclean) {
1011                 pe_rsc_debug(rsc, "Not locking %s to unclean %s for shutdown",
1012                              rsc->id, node->details->uname);
1013             } else {
1014                 rsc->lock_node = node;
1015                 rsc->lock_time = shutdown_time(node, data_set);
1016             }
1017         }
1018     }
1019 
1020     if (rsc->lock_node == NULL) {
1021         // No lock needed
1022         return;
1023     }
1024 
1025     if (data_set->shutdown_lock > 0) {
1026         time_t lock_expiration = rsc->lock_time + data_set->shutdown_lock;
1027 
1028         pe_rsc_info(rsc, "Locking %s to %s due to shutdown (expires @%lld)",
1029                     rsc->id, rsc->lock_node->details->uname,
1030                     (long long) lock_expiration);
1031         pe__update_recheck_time(++lock_expiration, data_set);
1032     } else {
1033         pe_rsc_info(rsc, "Locking %s to %s due to shutdown",
1034                     rsc->id, rsc->lock_node->details->uname);
1035     }
1036 
1037     // If resource is locked to one node, ban it from all other nodes
1038     for (GList *item = data_set->nodes; item != NULL; item = item->next) {
1039         pe_node_t *node = item->data;
1040 
1041         if (strcmp(node->details->uname, rsc->lock_node->details->uname)) {
1042             resource_location(rsc, node, -CRM_SCORE_INFINITY,
1043                               XML_CONFIG_ATTR_SHUTDOWN_LOCK, data_set);
1044         }
1045     }
1046 }
1047 
1048 /*
1049  * \internal
1050  * \brief Stage 2 of cluster status: apply node-specific criteria
1051  *
1052  * Count known nodes, and apply location constraints, stickiness, and exclusive
1053  * resource discovery.
1054  */
1055 gboolean
1056 stage2(pe_working_set_t * data_set)
     /*  */
1057 {
1058     GList *gIter = NULL;
1059 
1060     if (pcmk_is_set(data_set->flags, pe_flag_shutdown_lock)) {
1061         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1062             apply_shutdown_lock((pe_resource_t *) gIter->data, data_set);
1063         }
1064     }
1065 
1066     if (!pcmk_is_set(data_set->flags, pe_flag_no_compat)) {
1067         // @COMPAT API backward compatibility
1068         for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1069             pe_node_t *node = (pe_node_t *) gIter->data;
1070 
1071             if (node && (node->weight >= 0) && node->details->online
1072                 && (node->details->type != node_ping)) {
1073                 data_set->max_valid_nodes++;
1074             }
1075         }
1076     }
1077 
1078     apply_placement_constraints(data_set);
1079 
1080     gIter = data_set->nodes;
1081     for (; gIter != NULL; gIter = gIter->next) {
1082         GList *gIter2 = NULL;
1083         pe_node_t *node = (pe_node_t *) gIter->data;
1084 
1085         gIter2 = data_set->resources;
1086         for (; gIter2 != NULL; gIter2 = gIter2->next) {
1087             pe_resource_t *rsc = (pe_resource_t *) gIter2->data;
1088 
1089             common_apply_stickiness(rsc, node, data_set);
1090             rsc_discover_filter(rsc, node);
1091         }
1092     }
1093 
1094     return TRUE;
1095 }
1096 
1097 /*
1098  * Create internal resource constraints before allocation
1099  */
1100 gboolean
1101 stage3(pe_working_set_t * data_set)
     /*  */
1102 {
1103 
1104     GList *gIter = data_set->resources;
1105 
1106     for (; gIter != NULL; gIter = gIter->next) {
1107         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1108 
1109         rsc->cmds->internal_constraints(rsc, data_set);
1110     }
1111 
1112     return TRUE;
1113 }
1114 
1115 /*
1116  * Check for orphaned or redefined actions
1117  */
1118 gboolean
1119 stage4(pe_working_set_t * data_set)
     /*  */
1120 {
1121     check_actions(data_set);
1122     return TRUE;
1123 }
1124 
1125 static void *
1126 convert_const_pointer(const void *ptr)
     /*  */
1127 {
1128     /* Worst function ever */
1129     return (void *)ptr;
1130 }
1131 
1132 static gint
1133 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
     /*  */
1134 {
1135     int rc = 0;
1136     int r1_weight = -INFINITY;
1137     int r2_weight = -INFINITY;
1138 
1139     const char *reason = "existence";
1140 
1141     GList *nodes = (GList *) data;
1142     const pe_resource_t *resource1 = a;
1143     const pe_resource_t *resource2 = b;
1144 
1145     pe_node_t *r1_node = NULL;
1146     pe_node_t *r2_node = NULL;
1147     GList *gIter = NULL;
1148     GHashTable *r1_nodes = NULL;
1149     GHashTable *r2_nodes = NULL;
1150 
1151     reason = "priority";
1152     r1_weight = resource1->priority;
1153     r2_weight = resource2->priority;
1154 
1155     if (r1_weight > r2_weight) {
1156         rc = -1;
1157         goto done;
1158     }
1159 
1160     if (r1_weight < r2_weight) {
1161         rc = 1;
1162         goto done;
1163     }
1164 
1165     reason = "no node list";
1166     if (nodes == NULL) {
1167         goto done;
1168     }
1169 
1170     r1_nodes = pcmk__native_merge_weights(convert_const_pointer(resource1),
1171                                           resource1->id, NULL, NULL, 1,
1172                                           pe_weights_forward | pe_weights_init);
1173     pe__show_node_weights(true, NULL, resource1->id, r1_nodes,
1174                           resource1->cluster);
1175 
1176     r2_nodes = pcmk__native_merge_weights(convert_const_pointer(resource2),
1177                                           resource2->id, NULL, NULL, 1,
1178                                           pe_weights_forward | pe_weights_init);
1179     pe__show_node_weights(true, NULL, resource2->id, r2_nodes,
1180                           resource2->cluster);
1181 
1182     /* Current location score */
1183     reason = "current location";
1184     r1_weight = -INFINITY;
1185     r2_weight = -INFINITY;
1186 
1187     if (resource1->running_on) {
1188         r1_node = pe__current_node(resource1);
1189         r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
1190         if (r1_node != NULL) {
1191             r1_weight = r1_node->weight;
1192         }
1193     }
1194     if (resource2->running_on) {
1195         r2_node = pe__current_node(resource2);
1196         r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
1197         if (r2_node != NULL) {
1198             r2_weight = r2_node->weight;
1199         }
1200     }
1201 
1202     if (r1_weight > r2_weight) {
1203         rc = -1;
1204         goto done;
1205     }
1206 
1207     if (r1_weight < r2_weight) {
1208         rc = 1;
1209         goto done;
1210     }
1211 
1212     reason = "score";
1213     for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
1214         pe_node_t *node = (pe_node_t *) gIter->data;
1215 
1216         r1_node = NULL;
1217         r2_node = NULL;
1218 
1219         r1_weight = -INFINITY;
1220         if (r1_nodes) {
1221             r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
1222         }
1223         if (r1_node) {
1224             r1_weight = r1_node->weight;
1225         }
1226 
1227         r2_weight = -INFINITY;
1228         if (r2_nodes) {
1229             r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
1230         }
1231         if (r2_node) {
1232             r2_weight = r2_node->weight;
1233         }
1234 
1235         if (r1_weight > r2_weight) {
1236             rc = -1;
1237             goto done;
1238         }
1239 
1240         if (r1_weight < r2_weight) {
1241             rc = 1;
1242             goto done;
1243         }
1244     }
1245 
1246   done:
1247     crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
1248               resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
1249               rc < 0 ? '>' : rc > 0 ? '<' : '=',
1250               resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
1251 
1252     if (r1_nodes) {
1253         g_hash_table_destroy(r1_nodes);
1254     }
1255     if (r2_nodes) {
1256         g_hash_table_destroy(r2_nodes);
1257     }
1258 
1259     return rc;
1260 }
1261 
1262 static void
1263 allocate_resources(pe_working_set_t * data_set)
     /*  */
1264 {
1265     GList *gIter = NULL;
1266 
1267     if (pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
1268         /* Allocate remote connection resources first (which will also allocate
1269          * any colocation dependencies). If the connection is migrating, always
1270          * prefer the partial migration target.
1271          */
1272         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1273             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1274             if (rsc->is_remote_node == FALSE) {
1275                 continue;
1276             }
1277             pe_rsc_trace(rsc, "Allocating remote connection resource '%s'",
1278                          rsc->id);
1279             rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
1280         }
1281     }
1282 
1283     /* now do the rest of the resources */
1284     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1285         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1286         if (rsc->is_remote_node == TRUE) {
1287             continue;
1288         }
1289         pe_rsc_trace(rsc, "Allocating %s resource '%s'",
1290                      crm_element_name(rsc->xml), rsc->id);
1291         rsc->cmds->allocate(rsc, NULL, data_set);
1292     }
1293 }
1294 
1295 /* We always use pe_order_preserve with these convenience functions to exempt
1296  * internally generated constraints from the prohibition of user constraints
1297  * involving remote connection resources.
1298  *
1299  * The start ordering additionally uses pe_order_runnable_left so that the
1300  * specified action is not runnable if the start is not runnable.
1301  */
1302 
1303 static inline void
1304 order_start_then_action(pe_resource_t *lh_rsc, pe_action_t *rh_action,
     /*  */
1305                         enum pe_ordering extra, pe_working_set_t *data_set)
1306 {
1307     if (lh_rsc && rh_action && data_set) {
1308         custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
1309                             rh_action->rsc, NULL, rh_action,
1310                             pe_order_preserve | pe_order_runnable_left | extra,
1311                             data_set);
1312     }
1313 }
1314 
1315 static inline void
1316 order_action_then_stop(pe_action_t *lh_action, pe_resource_t *rh_rsc,
     /*  */
1317                        enum pe_ordering extra, pe_working_set_t *data_set)
1318 {
1319     if (lh_action && rh_rsc && data_set) {
1320         custom_action_order(lh_action->rsc, NULL, lh_action,
1321                             rh_rsc, stop_key(rh_rsc), NULL,
1322                             pe_order_preserve | extra, data_set);
1323     }
1324 }
1325 
1326 // Clear fail counts for orphaned rsc on all online nodes
1327 static void
1328 cleanup_orphans(pe_resource_t * rsc, pe_working_set_t * data_set)
     /*  */
1329 {
1330     GList *gIter = NULL;
1331 
1332     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1333         pe_node_t *node = (pe_node_t *) gIter->data;
1334 
1335         if (node->details->online
1336             && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
1337                                 data_set)) {
1338 
1339             pe_action_t *clear_op = NULL;
1340 
1341             clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
1342                                            data_set);
1343 
1344             /* We can't use order_action_then_stop() here because its
1345              * pe_order_preserve breaks things
1346              */
1347             custom_action_order(clear_op->rsc, NULL, clear_op,
1348                                 rsc, stop_key(rsc), NULL,
1349                                 pe_order_optional, data_set);
1350         }
1351     }
1352 }
1353 
1354 gboolean
1355 stage5(pe_working_set_t * data_set)
     /*  */
1356 {
1357     pcmk__output_t *out = data_set->priv;
1358     GList *gIter = NULL;
1359 
1360     if (!pcmk__str_eq(data_set->placement_strategy, "default", pcmk__str_casei)) {
1361         GList *nodes = g_list_copy(data_set->nodes);
1362 
1363         nodes = sort_nodes_by_weight(nodes, NULL, data_set);
1364         data_set->resources =
1365             g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
1366 
1367         g_list_free(nodes);
1368     }
1369 
1370     gIter = data_set->nodes;
1371     for (; gIter != NULL; gIter = gIter->next) {
1372         pe_node_t *node = (pe_node_t *) gIter->data;
1373 
1374         if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
1375             out->message(out, "node-capacity", node, "Original");
1376         }
1377     }
1378 
1379     crm_trace("Allocating services");
1380     /* Take (next) highest resource, assign it and create its actions */
1381 
1382     allocate_resources(data_set);
1383 
1384     gIter = data_set->nodes;
1385     for (; gIter != NULL; gIter = gIter->next) {
1386         pe_node_t *node = (pe_node_t *) gIter->data;
1387 
1388         if (pcmk_is_set(data_set->flags, pe_flag_show_utilization)) {
1389             out->message(out, "node-capacity", node, "Remaining");
1390         }
1391     }
1392 
1393     // Process deferred action checks
1394     pe__foreach_param_check(data_set, check_params);
1395     pe__free_param_checks(data_set);
1396 
1397     if (pcmk_is_set(data_set->flags, pe_flag_startup_probes)) {
1398         crm_trace("Calculating needed probes");
1399         /* This code probably needs optimization
1400          * ptest -x with 100 nodes, 100 clones and clone-max=100:
1401 
1402          With probes:
1403 
1404          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1405          ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1406          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1407          ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
1408          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1409          ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
1410          ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
1411          ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
1412          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
1413          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1414          ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1415          36s
1416          ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1417 
1418          Without probes:
1419 
1420          ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1421          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1422          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1423          ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
1424          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1425          ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
1426          ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
1427          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
1428          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1429          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1430          ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1431         */
1432 
1433         probe_resources(data_set);
1434     }
1435 
1436     crm_trace("Handle orphans");
1437     if (pcmk_is_set(data_set->flags, pe_flag_stop_rsc_orphans)) {
1438         for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1439             pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1440 
1441             /* There's no need to recurse into rsc->children because those
1442              * should just be unallocated clone instances.
1443              */
1444             if (pcmk_is_set(rsc->flags, pe_rsc_orphan)) {
1445                 cleanup_orphans(rsc, data_set);
1446             }
1447         }
1448     }
1449 
1450     crm_trace("Creating actions");
1451 
1452     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1453         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1454 
1455         rsc->cmds->create_actions(rsc, data_set);
1456     }
1457 
1458     crm_trace("Creating done");
1459     return TRUE;
1460 }
1461 
1462 static gboolean
1463 is_managed(const pe_resource_t * rsc)
     /*  */
1464 {
1465     GList *gIter = rsc->children;
1466 
1467     if (pcmk_is_set(rsc->flags, pe_rsc_managed)) {
1468         return TRUE;
1469     }
1470 
1471     for (; gIter != NULL; gIter = gIter->next) {
1472         pe_resource_t *child_rsc = (pe_resource_t *) gIter->data;
1473 
1474         if (is_managed(child_rsc)) {
1475             return TRUE;
1476         }
1477     }
1478 
1479     return FALSE;
1480 }
1481 
1482 static gboolean
1483 any_managed_resources(pe_working_set_t * data_set)
     /*  */
1484 {
1485 
1486     GList *gIter = data_set->resources;
1487 
1488     for (; gIter != NULL; gIter = gIter->next) {
1489         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
1490 
1491         if (is_managed(rsc)) {
1492             return TRUE;
1493         }
1494     }
1495     return FALSE;
1496 }
1497 
1498 /*!
1499  * \internal
1500  * \brief Create pseudo-op for guest node fence, and order relative to it
1501  *
1502  * \param[in] node      Guest node to fence
1503  * \param[in] data_set  Working set of CIB state
1504  */
1505 static void
1506 fence_guest(pe_node_t *node, pe_working_set_t *data_set)
     /*  */
1507 {
1508     pe_resource_t *container = node->details->remote_rsc->container;
1509     pe_action_t *stop = NULL;
1510     pe_action_t *stonith_op = NULL;
1511 
1512     /* The fence action is just a label; we don't do anything differently for
1513      * off vs. reboot. We specify it explicitly, rather than let it default to
1514      * cluster's default action, because we are not _initiating_ fencing -- we
1515      * are creating a pseudo-event to describe fencing that is already occurring
1516      * by other means (container recovery).
1517      */
1518     const char *fence_action = "off";
1519 
1520     /* Check whether guest's container resource has any explicit stop or
1521      * start (the stop may be implied by fencing of the guest's host).
1522      */
1523     if (container) {
1524         stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
1525 
1526         if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
1527             fence_action = "reboot";
1528         }
1529     }
1530 
1531     /* Create a fence pseudo-event, so we have an event to order actions
1532      * against, and the controller can always detect it.
1533      */
1534     stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", FALSE, data_set);
1535     pe__set_action_flags(stonith_op, pe_action_pseudo|pe_action_runnable);
1536 
1537     /* We want to imply stops/demotes after the guest is stopped, not wait until
1538      * it is restarted, so we always order pseudo-fencing after stop, not start
1539      * (even though start might be closer to what is done for a real reboot).
1540      */
1541     if ((stop != NULL) && pcmk_is_set(stop->flags, pe_action_pseudo)) {
1542         pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, FALSE, data_set);
1543         crm_info("Implying guest node %s is down (action %d) after %s fencing",
1544                  node->details->uname, stonith_op->id, stop->node->details->uname);
1545         order_actions(parent_stonith_op, stonith_op,
1546                       pe_order_runnable_left|pe_order_implies_then);
1547 
1548     } else if (stop) {
1549         order_actions(stop, stonith_op,
1550                       pe_order_runnable_left|pe_order_implies_then);
1551         crm_info("Implying guest node %s is down (action %d) "
1552                  "after container %s is stopped (action %d)",
1553                  node->details->uname, stonith_op->id,
1554                  container->id, stop->id);
1555     } else {
1556         /* If we're fencing the guest node but there's no stop for the guest
1557          * resource, we must think the guest is already stopped. However, we may
1558          * think so because its resource history was just cleaned. To avoid
1559          * unnecessarily considering the guest node down if it's really up,
1560          * order the pseudo-fencing after any stop of the connection resource,
1561          * which will be ordered after any container (re-)probe.
1562          */
1563         stop = find_first_action(node->details->remote_rsc->actions, NULL,
1564                                  RSC_STOP, NULL);
1565 
1566         if (stop) {
1567             order_actions(stop, stonith_op, pe_order_optional);
1568             crm_info("Implying guest node %s is down (action %d) "
1569                      "after connection is stopped (action %d)",
1570                      node->details->uname, stonith_op->id, stop->id);
1571         } else {
1572             /* Not sure why we're fencing, but everything must already be
1573              * cleanly stopped.
1574              */
1575             crm_info("Implying guest node %s is down (action %d) ",
1576                      node->details->uname, stonith_op->id);
1577         }
1578     }
1579 
1580     /* Order/imply other actions relative to pseudo-fence as with real fence */
1581     pcmk__order_vs_fence(stonith_op, data_set);
1582 }
1583 
1584 /*
1585  * Create dependencies for stonith and shutdown operations
1586  */
1587 gboolean
1588 stage6(pe_working_set_t * data_set)
     /*  */
1589 {
1590     pe_action_t *dc_down = NULL;
1591     pe_action_t *stonith_op = NULL;
1592     gboolean integrity_lost = FALSE;
1593     gboolean need_stonith = TRUE;
1594     GList *gIter;
1595     GList *stonith_ops = NULL;
1596     GList *shutdown_ops = NULL;
1597 
1598     /* Remote ordering constraints need to happen prior to calculating fencing
1599      * because it is one more place we will mark the node as dirty.
1600      *
1601      * A nice side effect of doing them early is that apply_*_ordering() can be
1602      * simpler because pe_fence_node() has already done some of the work.
1603      */
1604     crm_trace("Creating remote ordering constraints");
1605     apply_remote_node_ordering(data_set);
1606 
1607     crm_trace("Processing fencing and shutdown cases");
1608     if (any_managed_resources(data_set) == FALSE) {
1609         crm_notice("Delaying fencing operations until there are resources to manage");
1610         need_stonith = FALSE;
1611     }
1612 
1613     /* Check each node for stonith/shutdown */
1614     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1615         pe_node_t *node = (pe_node_t *) gIter->data;
1616 
1617         /* Guest nodes are "fenced" by recovering their container resource,
1618          * so handle them separately.
1619          */
1620         if (pe__is_guest_node(node)) {
1621             if (node->details->remote_requires_reset && need_stonith
1622                 && pe_can_fence(data_set, node)) {
1623                 fence_guest(node, data_set);
1624             }
1625             continue;
1626         }
1627 
1628         stonith_op = NULL;
1629 
1630         if (node->details->unclean
1631             && need_stonith && pe_can_fence(data_set, node)) {
1632 
1633             stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", FALSE, data_set);
1634             pe_warn("Scheduling Node %s for STONITH", node->details->uname);
1635 
1636             pcmk__order_vs_fence(stonith_op, data_set);
1637 
1638             if (node->details->is_dc) {
1639                 // Remember if the DC is being fenced
1640                 dc_down = stonith_op;
1641 
1642             } else {
1643 
1644                 if (!pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)
1645                     && (stonith_ops != NULL)) {
1646                     /* Concurrent fencing is disabled, so order each non-DC
1647                      * fencing in a chain. If there is any DC fencing or
1648                      * shutdown, it will be ordered after the last action in the
1649                      * chain later.
1650                      */
1651                     order_actions((pe_action_t *) stonith_ops->data,
1652                                   stonith_op, pe_order_optional);
1653                 }
1654 
1655                 // Remember all non-DC fencing actions in a separate list
1656                 stonith_ops = g_list_prepend(stonith_ops, stonith_op);
1657             }
1658 
1659         } else if (node->details->online && node->details->shutdown &&
1660                 /* TODO define what a shutdown op means for a remote node.
1661                  * For now we do not send shutdown operations for remote nodes, but
1662                  * if we can come up with a good use for this in the future, we will. */
1663                     pe__is_guest_or_remote_node(node) == FALSE) {
1664 
1665             pe_action_t *down_op = sched_shutdown_op(node, data_set);
1666 
1667             if (node->details->is_dc) {
1668                 // Remember if the DC is being shut down
1669                 dc_down = down_op;
1670             } else {
1671                 // Remember non-DC shutdowns for later ordering
1672                 shutdown_ops = g_list_prepend(shutdown_ops, down_op);
1673             }
1674         }
1675 
1676         if (node->details->unclean && stonith_op == NULL) {
1677             integrity_lost = TRUE;
1678             pe_warn("Node %s is unclean!", node->details->uname);
1679         }
1680     }
1681 
1682     if (integrity_lost) {
1683         if (!pcmk_is_set(data_set->flags, pe_flag_stonith_enabled)) {
1684             pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
1685             pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
1686 
1687         } else if (!pcmk_is_set(data_set->flags, pe_flag_have_quorum)) {
1688             crm_notice("Cannot fence unclean nodes until quorum is"
1689                        " attained (or no-quorum-policy is set to ignore)");
1690         }
1691     }
1692 
1693     if (dc_down != NULL) {
1694         /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
1695          * DC elections. However, we don't want to order non-DC shutdowns before
1696          * a DC *fencing*, because even though we don't want a node that's
1697          * shutting down to become DC, the DC fencing could be ordered before a
1698          * clone stop that's also ordered before the shutdowns, thus leading to
1699          * a graph loop.
1700          */
1701         if (pcmk__str_eq(dc_down->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
1702             for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
1703                 pe_action_t *node_stop = (pe_action_t *) gIter->data;
1704 
1705                 crm_debug("Ordering shutdown on %s before %s on DC %s",
1706                           node_stop->node->details->uname,
1707                           dc_down->task, dc_down->node->details->uname);
1708 
1709                 order_actions(node_stop, dc_down, pe_order_optional);
1710             }
1711         }
1712 
1713         // Order any non-DC fencing before any DC fencing or shutdown
1714 
1715         if (pcmk_is_set(data_set->flags, pe_flag_concurrent_fencing)) {
1716             /* With concurrent fencing, order each non-DC fencing action
1717              * separately before any DC fencing or shutdown.
1718              */
1719             for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
1720                 order_actions((pe_action_t *) gIter->data, dc_down,
1721                               pe_order_optional);
1722             }
1723         } else if (stonith_ops) {
1724             /* Without concurrent fencing, the non-DC fencing actions are
1725              * already ordered relative to each other, so we just need to order
1726              * the DC fencing after the last action in the chain (which is the
1727              * first item in the list).
1728              */
1729             order_actions((pe_action_t *) stonith_ops->data, dc_down,
1730                           pe_order_optional);
1731         }
1732     }
1733     g_list_free(stonith_ops);
1734     g_list_free(shutdown_ops);
1735     return TRUE;
1736 }
1737 
1738 /*
1739  * Determine the sets of independent actions and the correct order for the
1740  *  actions in each set.
1741  *
1742  * Mark dependencies of un-runnable actions un-runnable
1743  *
1744  */
1745 static GList *
1746 find_actions_by_task(GList *actions, pe_resource_t * rsc, const char *original_key)
     /*  */
1747 {
1748     GList *list = NULL;
1749 
1750     list = find_actions(actions, original_key, NULL);
1751     if (list == NULL) {
1752         /* we're potentially searching a child of the original resource */
1753         char *key = NULL;
1754         char *task = NULL;
1755         guint interval_ms = 0;
1756 
1757         if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
1758             key = pcmk__op_key(rsc->id, task, interval_ms);
1759             list = find_actions(actions, key, NULL);
1760 
1761         } else {
1762             crm_err("search key: %s", original_key);
1763         }
1764 
1765         free(key);
1766         free(task);
1767     }
1768 
1769     return list;
1770 }
1771 
1772 static void
1773 rsc_order_then(pe_action_t *lh_action, pe_resource_t *rsc,
     /*  */
1774                pe__ordering_t *order)
1775 {
1776     GList *gIter = NULL;
1777     GList *rh_actions = NULL;
1778     pe_action_t *rh_action = NULL;
1779     enum pe_ordering type;
1780 
1781     CRM_CHECK(rsc != NULL, return);
1782     CRM_CHECK(order != NULL, return);
1783 
1784     type = order->type;
1785     rh_action = order->rh_action;
1786     crm_trace("Processing RH of ordering constraint %d", order->id);
1787 
1788     if (rh_action != NULL) {
1789         rh_actions = g_list_prepend(NULL, rh_action);
1790 
1791     } else if (rsc != NULL) {
1792         rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
1793     }
1794 
1795     if (rh_actions == NULL) {
1796         pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
1797                      " ignoring", rsc->id, order->rh_action_task);
1798         if (lh_action) {
1799             pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
1800         }
1801         return;
1802     }
1803 
1804     if ((lh_action != NULL) && (lh_action->rsc == rsc)
1805         && pcmk_is_set(lh_action->flags, pe_action_dangle)) {
1806 
1807         pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
1808                      order->rh_action_task);
1809         pe__clear_order_flags(type, pe_order_implies_then);
1810     }
1811 
1812     gIter = rh_actions;
1813     for (; gIter != NULL; gIter = gIter->next) {
1814         pe_action_t *rh_action_iter = (pe_action_t *) gIter->data;
1815 
1816         if (lh_action) {
1817             order_actions(lh_action, rh_action_iter, type);
1818 
1819         } else if (type & pe_order_implies_then) {
1820             pe__clear_action_flags(rh_action_iter, pe_action_runnable);
1821             crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
1822         } else {
1823             crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
1824         }
1825     }
1826 
1827     g_list_free(rh_actions);
1828 }
1829 
1830 static void
1831 rsc_order_first(pe_resource_t *lh_rsc, pe__ordering_t *order,
     /*  */
1832                 pe_working_set_t *data_set)
1833 {
1834     GList *gIter = NULL;
1835     GList *lh_actions = NULL;
1836     pe_action_t *lh_action = order->lh_action;
1837     pe_resource_t *rh_rsc = order->rh_rsc;
1838 
1839     crm_trace("Processing LH of ordering constraint %d", order->id);
1840     CRM_ASSERT(lh_rsc != NULL);
1841 
1842     if (lh_action != NULL) {
1843         lh_actions = g_list_prepend(NULL, lh_action);
1844 
1845     } else {
1846         lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
1847     }
1848 
1849     if (lh_actions == NULL && lh_rsc != rh_rsc) {
1850         char *key = NULL;
1851         char *op_type = NULL;
1852         guint interval_ms = 0;
1853 
1854         parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
1855         key = pcmk__op_key(lh_rsc->id, op_type, interval_ms);
1856 
1857         if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && pcmk__str_eq(op_type, RSC_STOP, pcmk__str_casei)) {
1858             free(key);
1859             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1860                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1861 
1862         } else if ((lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_UNPROMOTED)
1863                    && pcmk__str_eq(op_type, RSC_DEMOTE, pcmk__str_casei)) {
1864             free(key);
1865             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1866                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1867 
1868         } else {
1869             pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
1870                          lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1871             lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
1872             lh_actions = g_list_prepend(NULL, lh_action);
1873         }
1874 
1875         free(op_type);
1876     }
1877 
1878     gIter = lh_actions;
1879     for (; gIter != NULL; gIter = gIter->next) {
1880         pe_action_t *lh_action_iter = (pe_action_t *) gIter->data;
1881 
1882         if (rh_rsc == NULL && order->rh_action) {
1883             rh_rsc = order->rh_action->rsc;
1884         }
1885         if (rh_rsc) {
1886             rsc_order_then(lh_action_iter, rh_rsc, order);
1887 
1888         } else if (order->rh_action) {
1889             order_actions(lh_action_iter, order->rh_action, order->type);
1890         }
1891     }
1892 
1893     g_list_free(lh_actions);
1894 }
1895 
1896 extern void update_colo_start_chain(pe_action_t *action,
1897                                     pe_working_set_t *data_set);
1898 
1899 static int
1900 is_recurring_action(pe_action_t *action)
     /*  */
1901 {
1902     guint interval_ms;
1903 
1904     if (pcmk__guint_from_hash(action->meta,
1905                               XML_LRM_ATTR_INTERVAL_MS, 0,
1906                               &interval_ms) != pcmk_rc_ok) {
1907         return 0;
1908     }
1909     return (interval_ms > 0);
1910 }
1911 
1912 static void
1913 apply_container_ordering(pe_action_t *action, pe_working_set_t *data_set)
     /*  */
1914 {
1915     /* VMs are also classified as containers for these purposes... in
1916      * that they both involve a 'thing' running on a real or remote
1917      * cluster node.
1918      *
1919      * This allows us to be smarter about the type and extent of
1920      * recovery actions required in various scenarios
1921      */
1922     pe_resource_t *remote_rsc = NULL;
1923     pe_resource_t *container = NULL;
1924     enum action_tasks task = text2task(action->task);
1925 
1926     CRM_ASSERT(action->rsc);
1927     CRM_ASSERT(action->node);
1928     CRM_ASSERT(pe__is_guest_or_remote_node(action->node));
1929 
1930     remote_rsc = action->node->details->remote_rsc;
1931     CRM_ASSERT(remote_rsc);
1932 
1933     container = remote_rsc->container;
1934     CRM_ASSERT(container);
1935 
1936     if (pcmk_is_set(container->flags, pe_rsc_failed)) {
1937         pe_fence_node(data_set, action->node, "container failed", FALSE);
1938     }
1939 
1940     crm_trace("Order %s action %s relative to %s%s for %s%s",
1941               action->task, action->uuid,
1942               pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
1943               remote_rsc->id,
1944               pcmk_is_set(container->flags, pe_rsc_failed)? "failed " : "",
1945               container->id);
1946 
1947     if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
1948         /* Migration ops map to "no_action", but we need to apply the same
1949          * ordering as for stop or demote (see get_router_node()).
1950          */
1951         task = stop_rsc;
1952     }
1953 
1954     switch (task) {
1955         case start_rsc:
1956         case action_promote:
1957             /* Force resource recovery if the container is recovered */
1958             order_start_then_action(container, action, pe_order_implies_then,
1959                                     data_set);
1960 
1961             /* Wait for the connection resource to be up too */
1962             order_start_then_action(remote_rsc, action, pe_order_none,
1963                                     data_set);
1964             break;
1965 
1966         case stop_rsc:
1967         case action_demote:
1968             if (pcmk_is_set(container->flags, pe_rsc_failed)) {
1969                 /* When the container representing a guest node fails, any stop
1970                  * or demote actions for resources running on the guest node
1971                  * are implied by the container stopping. This is similar to
1972                  * how fencing operations work for cluster nodes and remote
1973                  * nodes.
1974                  */
1975             } else {
1976                 /* Ensure the operation happens before the connection is brought
1977                  * down.
1978                  *
1979                  * If we really wanted to, we could order these after the
1980                  * connection start, IFF the container's current role was
1981                  * stopped (otherwise we re-introduce an ordering loop when the
1982                  * connection is restarting).
1983                  */
1984                 order_action_then_stop(action, remote_rsc, pe_order_none,
1985                                        data_set);
1986             }
1987             break;
1988 
1989         default:
1990             /* Wait for the connection resource to be up */
1991             if (is_recurring_action(action)) {
1992                 /* In case we ever get the recovery logic wrong, force
1993                  * recurring monitors to be restarted, even if just
1994                  * the connection was re-established
1995                  */
1996                 if(task != no_action) {
1997                     order_start_then_action(remote_rsc, action,
1998                                             pe_order_implies_then, data_set);
1999                 }
2000             } else {
2001                 order_start_then_action(remote_rsc, action, pe_order_none,
2002                                         data_set);
2003             }
2004             break;
2005     }
2006 }
2007 
2008 static enum remote_connection_state
2009 get_remote_node_state(pe_node_t *node) 
     /*  */
2010 {
2011     pe_resource_t *remote_rsc = NULL;
2012     pe_node_t *cluster_node = NULL;
2013 
2014     CRM_ASSERT(node);
2015 
2016     remote_rsc = node->details->remote_rsc;
2017     CRM_ASSERT(remote_rsc);
2018 
2019     cluster_node = pe__current_node(remote_rsc);
2020 
2021     /* If the cluster node the remote connection resource resides on
2022      * is unclean or went offline, we can't process any operations
2023      * on that remote node until after it starts elsewhere.
2024      */
2025     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
2026         /* The connection resource is not going to run anywhere */
2027 
2028         if (cluster_node && cluster_node->details->unclean) {
2029             /* The remote connection is failed because its resource is on a
2030              * failed node and can't be recovered elsewhere, so we must fence.
2031              */
2032             return remote_state_failed;
2033         }
2034 
2035         if (!pcmk_is_set(remote_rsc->flags, pe_rsc_failed)) {
2036             /* Connection resource is cleanly stopped */
2037             return remote_state_stopped;
2038         }
2039 
2040         /* Connection resource is failed */
2041 
2042         if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
2043             && remote_rsc->remote_reconnect_ms
2044             && node->details->remote_was_fenced
2045             && !pe__shutdown_requested(node)) {
2046 
2047             /* We won't know whether the connection is recoverable until the
2048              * reconnect interval expires and we reattempt connection.
2049              */
2050             return remote_state_unknown;
2051         }
2052 
2053         /* The remote connection is in a failed state. If there are any
2054          * resources known to be active on it (stop) or in an unknown state
2055          * (probe), we must assume the worst and fence it.
2056          */
2057         return remote_state_failed;
2058 
2059     } else if (cluster_node == NULL) {
2060         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
2061         return remote_state_unknown;
2062 
2063     } else if(cluster_node->details->unclean == TRUE
2064               || cluster_node->details->online == FALSE) {
2065         /* Connection is running on a dead node, see if we can recover it first */
2066         return remote_state_resting;
2067 
2068     } else if (pcmk__list_of_multiple(remote_rsc->running_on)
2069                && remote_rsc->partial_migration_source
2070                && remote_rsc->partial_migration_target) {
2071         /* We're in the middle of migrating a connection resource,
2072          * wait until after the resource migrates before performing
2073          * any actions.
2074          */
2075         return remote_state_resting;
2076 
2077     }
2078     return remote_state_alive;
2079 }
2080 
2081 /*!
2082  * \internal
2083  * \brief Order actions on remote node relative to actions for the connection
2084  */
2085 static void
2086 apply_remote_ordering(pe_action_t *action, pe_working_set_t *data_set)
     /*  */
2087 {
2088     pe_resource_t *remote_rsc = NULL;
2089     enum action_tasks task = text2task(action->task);
2090     enum remote_connection_state state = get_remote_node_state(action->node);
2091 
2092     enum pe_ordering order_opts = pe_order_none;
2093 
2094     if (action->rsc == NULL) {
2095         return;
2096     }
2097 
2098     CRM_ASSERT(action->node);
2099     CRM_ASSERT(pe__is_guest_or_remote_node(action->node));
2100 
2101     remote_rsc = action->node->details->remote_rsc;
2102     CRM_ASSERT(remote_rsc);
2103 
2104     crm_trace("Order %s action %s relative to %s%s (state: %s)",
2105               action->task, action->uuid,
2106               pcmk_is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
2107               remote_rsc->id, state2text(state));
2108 
2109     if (pcmk__strcase_any_of(action->task, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
2110         /* Migration ops map to "no_action", but we need to apply the same
2111          * ordering as for stop or demote (see get_router_node()).
2112          */
2113         task = stop_rsc;
2114     }
2115 
2116     switch (task) {
2117         case start_rsc:
2118         case action_promote:
2119             order_opts = pe_order_none;
2120 
2121             if (state == remote_state_failed) {
2122                 /* Force recovery, by making this action required */
2123                 pe__set_order_flags(order_opts, pe_order_implies_then);
2124             }
2125 
2126             /* Ensure connection is up before running this action */
2127             order_start_then_action(remote_rsc, action, order_opts, data_set);
2128             break;
2129 
2130         case stop_rsc:
2131             if(state == remote_state_alive) {
2132                 order_action_then_stop(action, remote_rsc,
2133                                        pe_order_implies_first, data_set);
2134 
2135             } else if(state == remote_state_failed) {
2136                 /* The resource is active on the node, but since we don't have a
2137                  * valid connection, the only way to stop the resource is by
2138                  * fencing the node. There is no need to order the stop relative
2139                  * to the remote connection, since the stop will become implied
2140                  * by the fencing.
2141                  */
2142                 pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable", FALSE);
2143 
2144             } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
2145                 /* State must be remote_state_unknown or remote_state_stopped.
2146                  * Since the connection is not coming back up in this
2147                  * transition, stop this resource first.
2148                  */
2149                 order_action_then_stop(action, remote_rsc,
2150                                        pe_order_implies_first, data_set);
2151 
2152             } else {
2153                 /* The connection is going to be started somewhere else, so
2154                  * stop this resource after that completes.
2155                  */
2156                 order_start_then_action(remote_rsc, action, pe_order_none, data_set);
2157             }
2158             break;
2159 
2160         case action_demote:
2161             /* Only order this demote relative to the connection start if the
2162              * connection isn't being torn down. Otherwise, the demote would be
2163              * blocked because the connection start would not be allowed.
2164              */
2165             if(state == remote_state_resting || state == remote_state_unknown) {
2166                 order_start_then_action(remote_rsc, action, pe_order_none,
2167                                         data_set);
2168             } /* Otherwise we can rely on the stop ordering */
2169             break;
2170 
2171         default:
2172             /* Wait for the connection resource to be up */
2173             if (is_recurring_action(action)) {
2174                 /* In case we ever get the recovery logic wrong, force
2175                  * recurring monitors to be restarted, even if just
2176                  * the connection was re-established
2177                  */
2178                 order_start_then_action(remote_rsc, action,
2179                                         pe_order_implies_then, data_set);
2180 
2181             } else {
2182                 pe_node_t *cluster_node = pe__current_node(remote_rsc);
2183 
2184                 if(task == monitor_rsc && state == remote_state_failed) {
2185                     /* We would only be here if we do not know the
2186                      * state of the resource on the remote node.
2187                      * Since we have no way to find out, it is
2188                      * necessary to fence the node.
2189                      */
2190                     pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable", FALSE);
2191                 }
2192 
2193                 if(cluster_node && state == remote_state_stopped) {
2194                     /* The connection is currently up, but is going
2195                      * down permanently.
2196                      *
2197                      * Make sure we check services are actually
2198                      * stopped _before_ we let the connection get
2199                      * closed
2200                      */
2201                     order_action_then_stop(action, remote_rsc,
2202                                            pe_order_runnable_left, data_set);
2203 
2204                 } else {
2205                     order_start_then_action(remote_rsc, action, pe_order_none,
2206                                             data_set);
2207                 }
2208             }
2209             break;
2210     }
2211 }
2212 
2213 static void
2214 apply_remote_node_ordering(pe_working_set_t *data_set)
     /*  */
2215 {
2216     if (!pcmk_is_set(data_set->flags, pe_flag_have_remote_nodes)) {
2217         return;
2218     }
2219 
2220     for (GList *gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2221         pe_action_t *action = (pe_action_t *) gIter->data;
2222         pe_resource_t *remote = NULL;
2223 
2224         // We are only interested in resource actions
2225         if (action->rsc == NULL) {
2226             continue;
2227         }
2228 
2229         /* Special case: If we are clearing the failcount of an actual
2230          * remote connection resource, then make sure this happens before
2231          * any start of the resource in this transition.
2232          */
2233         if (action->rsc->is_remote_node &&
2234             pcmk__str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT, pcmk__str_casei)) {
2235 
2236             custom_action_order(action->rsc,
2237                 NULL,
2238                 action,
2239                 action->rsc,
2240                 pcmk__op_key(action->rsc->id, RSC_START, 0),
2241                 NULL,
2242                 pe_order_optional,
2243                 data_set);
2244 
2245             continue;
2246         }
2247 
2248         // We are only interested in actions allocated to a node
2249         if (action->node == NULL) {
2250             continue;
2251         }
2252 
2253         if (!pe__is_guest_or_remote_node(action->node)) {
2254             continue;
2255         }
2256 
2257         /* We are only interested in real actions.
2258          *
2259          * @TODO This is probably wrong; pseudo-actions might be converted to
2260          * real actions and vice versa later in update_actions() at the end of
2261          * stage7().
2262          */
2263         if (pcmk_is_set(action->flags, pe_action_pseudo)) {
2264             continue;
2265         }
2266 
2267         remote = action->node->details->remote_rsc;
2268         if (remote == NULL) {
2269             // Orphaned
2270             continue;
2271         }
2272 
2273         /* Another special case: if a resource is moving to a Pacemaker Remote
2274          * node, order the stop on the original node after any start of the
2275          * remote connection. This ensures that if the connection fails to
2276          * start, we leave the resource running on the original node.
2277          */
2278         if (pcmk__str_eq(action->task, RSC_START, pcmk__str_casei)) {
2279             for (GList *item = action->rsc->actions; item != NULL;
2280                  item = item->next) {
2281                 pe_action_t *rsc_action = item->data;
2282 
2283                 if ((rsc_action->node->details != action->node->details)
2284                     && pcmk__str_eq(rsc_action->task, RSC_STOP, pcmk__str_casei)) {
2285                     custom_action_order(remote, start_key(remote), NULL,
2286                                         action->rsc, NULL, rsc_action,
2287                                         pe_order_optional, data_set);
2288                 }
2289             }
2290         }
2291 
2292         /* The action occurs across a remote connection, so create
2293          * ordering constraints that guarantee the action occurs while the node
2294          * is active (after start, before stop ... things like that).
2295          *
2296          * This is somewhat brittle in that we need to make sure the results of
2297          * this ordering are compatible with the result of get_router_node().
2298          * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
2299          * of this logic rather than action2xml().
2300          */
2301         if (remote->container) {
2302             crm_trace("Container ordering for %s", action->uuid);
2303             apply_container_ordering(action, data_set);
2304 
2305         } else {
2306             crm_trace("Remote ordering for %s", action->uuid);
2307             apply_remote_ordering(action, data_set);
2308         }
2309     }
2310 }
2311 
2312 static gboolean
2313 order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
     /*  */
2314 {
2315     /* No need to probe the resource on the node that is being
2316      * unfenced. Otherwise it might introduce transition loop
2317      * since probe will be performed after the node is
2318      * unfenced.
2319      */
2320     if (pcmk__str_eq(rh_action->task, CRM_OP_FENCE, pcmk__str_casei)
2321          && probe->node && rh_action->node
2322          && probe->node->details == rh_action->node->details) {
2323         const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");
2324 
2325         if (pcmk__str_eq(op, "on", pcmk__str_casei)) {
2326             return TRUE;
2327         }
2328     }
2329 
2330     // Shutdown waits for probe to complete only if it's on the same node
2331     if ((pcmk__str_eq(rh_action->task, CRM_OP_SHUTDOWN, pcmk__str_casei))
2332         && probe->node && rh_action->node
2333         && probe->node->details != rh_action->node->details) {
2334         return TRUE;
2335     }
2336     return FALSE;
2337 }
2338 
2339 static void
2340 order_first_probes_imply_stops(pe_working_set_t * data_set)
     /*  */
2341 {
2342     GList *gIter = NULL;
2343 
2344     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
2345         pe__ordering_t *order = gIter->data;
2346         enum pe_ordering order_type = pe_order_optional;
2347 
2348         pe_resource_t *lh_rsc = order->lh_rsc;
2349         pe_resource_t *rh_rsc = order->rh_rsc;
2350         pe_action_t *lh_action = order->lh_action;
2351         pe_action_t *rh_action = order->rh_action;
2352         const char *lh_action_task = order->lh_action_task;
2353         const char *rh_action_task = order->rh_action_task;
2354 
2355         GList *probes = NULL;
2356         GList *rh_actions = NULL;
2357 
2358         GList *pIter = NULL;
2359 
2360         if (lh_rsc == NULL) {
2361             continue;
2362 
2363         } else if (rh_rsc && lh_rsc == rh_rsc) {
2364             continue;
2365         }
2366 
2367         if (lh_action == NULL && lh_action_task == NULL) {
2368             continue;
2369         }
2370 
2371         if (rh_action == NULL && rh_action_task == NULL) {
2372             continue;
2373         }
2374 
2375         /* Technically probe is expected to return "not running", which could be
2376          * the alternative of stop action if the status of the resource is
2377          * unknown yet.
2378          */
2379         if (lh_action && !pcmk__str_eq(lh_action->task, RSC_STOP, pcmk__str_casei)) {
2380             continue;
2381 
2382         } else if (lh_action == NULL
2383                    && lh_action_task
2384                    && !pcmk__ends_with(lh_action_task, "_" RSC_STOP "_0")) {
2385             continue;
2386         }
2387 
2388         /* Do not probe the resource inside of a stopping container. Otherwise
2389          * it might introduce transition loop since probe will be performed
2390          * after the container starts again.
2391          */
2392         if (rh_rsc && lh_rsc->container == rh_rsc) {
2393             if (rh_action && pcmk__str_eq(rh_action->task, RSC_STOP, pcmk__str_casei)) {
2394                 continue;
2395 
2396             } else if (rh_action == NULL && rh_action_task
2397                        && pcmk__ends_with(rh_action_task,"_" RSC_STOP "_0")) {
2398                 continue;
2399             }
2400         }
2401 
2402         if (order->type == pe_order_none) {
2403             continue;
2404         }
2405 
2406         // Preserve the order options for future filtering
2407         if (pcmk_is_set(order->type, pe_order_apply_first_non_migratable)) {
2408             pe__set_order_flags(order_type,
2409                                 pe_order_apply_first_non_migratable);
2410         }
2411 
2412         if (pcmk_is_set(order->type, pe_order_same_node)) {
2413             pe__set_order_flags(order_type, pe_order_same_node);
2414         }
2415 
2416         // Keep the order types for future filtering
2417         if (order->type == pe_order_anti_colocation
2418                    || order->type == pe_order_load) {
2419             order_type = order->type;
2420         }
2421 
2422         probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
2423         if (probes == NULL) {
2424             continue;
2425         }
2426 
2427         if (rh_action) {
2428             rh_actions = g_list_prepend(rh_actions, rh_action);
2429 
2430         } else if (rh_rsc && rh_action_task) {
2431             rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
2432         }
2433 
2434         if (rh_actions == NULL) {
2435             g_list_free(probes);
2436             continue;
2437         }
2438 
2439         crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
2440                   " (id=%d, type=%.6x)",
2441                   lh_action ? lh_action->uuid : lh_action_task,
2442                   rh_action ? rh_action->uuid : rh_action_task,
2443                   order->id, order->type);
2444 
2445         for (pIter = probes; pIter != NULL; pIter = pIter->next) {
2446             pe_action_t *probe = (pe_action_t *) pIter->data;
2447             GList *rIter = NULL;
2448 
2449             for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
2450                 pe_action_t *rh_action_iter = (pe_action_t *) rIter->data;
2451 
2452                 if (order_first_probe_unneeded(probe, rh_action_iter)) {
2453                     continue;
2454                 }
2455                 order_actions(probe, rh_action_iter, order_type);
2456             }
2457         }
2458 
2459         g_list_free(rh_actions);
2460         g_list_free(probes);
2461     }
2462 }
2463 
2464 static void
2465 order_first_probe_then_restart_repromote(pe_action_t * probe,
     /*  */
2466                                          pe_action_t * after,
2467                                          pe_working_set_t * data_set)
2468 {
2469     GList *gIter = NULL;
2470     bool interleave = FALSE;
2471     pe_resource_t *compatible_rsc = NULL;
2472 
2473     if (probe == NULL
2474         || probe->rsc == NULL
2475         || probe->rsc->variant != pe_native) {
2476         return;
2477     }
2478 
2479     if (after == NULL
2480         // Avoid running into any possible loop
2481         || pcmk_is_set(after->flags, pe_action_tracking)) {
2482         return;
2483     }
2484 
2485     if (!pcmk__str_eq(probe->task, RSC_STATUS, pcmk__str_casei)) {
2486         return;
2487     }
2488 
2489     pe__set_action_flags(after, pe_action_tracking);
2490 
2491     crm_trace("Processing based on %s %s -> %s %s",
2492               probe->uuid,
2493               probe->node ? probe->node->details->uname: "",
2494               after->uuid,
2495               after->node ? after->node->details->uname : "");
2496 
2497     if (after->rsc
2498         /* Better not build a dependency directly with a clone/group.
2499          * We are going to proceed through the ordering chain and build
2500          * dependencies with its children.
2501          */
2502         && after->rsc->variant == pe_native
2503         && probe->rsc != after->rsc) {
2504 
2505             GList *then_actions = NULL;
2506             enum pe_ordering probe_order_type = pe_order_optional;
2507 
2508             if (pcmk__str_eq(after->task, RSC_START, pcmk__str_casei)) {
2509                 then_actions = pe__resource_actions(after->rsc, NULL, RSC_STOP, FALSE);
2510 
2511             } else if (pcmk__str_eq(after->task, RSC_PROMOTE, pcmk__str_casei)) {
2512                 then_actions = pe__resource_actions(after->rsc, NULL, RSC_DEMOTE, FALSE);
2513             }
2514 
2515             for (gIter = then_actions; gIter != NULL; gIter = gIter->next) {
2516                 pe_action_t *then = (pe_action_t *) gIter->data;
2517 
2518                 // Skip any pseudo action which for example is implied by fencing
2519                 if (pcmk_is_set(then->flags, pe_action_pseudo)) {
2520                     continue;
2521                 }
2522 
2523                 order_actions(probe, then, probe_order_type);
2524             }
2525             g_list_free(then_actions);
2526     }
2527 
2528     if (after->rsc
2529         && after->rsc->variant > pe_group) {
2530         const char *interleave_s = g_hash_table_lookup(after->rsc->meta,
2531                                                        XML_RSC_ATTR_INTERLEAVE);
2532 
2533         interleave = crm_is_true(interleave_s);
2534 
2535         if (interleave) {
2536             /* For an interleaved clone, we should build a dependency only
2537              * with the relevant clone child.
2538              */
2539             compatible_rsc = find_compatible_child(probe->rsc,
2540                                                    after->rsc,
2541                                                    RSC_ROLE_UNKNOWN,
2542                                                    FALSE, data_set);
2543         }
2544     }
2545 
2546     for (gIter = after->actions_after; gIter != NULL; gIter = gIter->next) {
2547         pe_action_wrapper_t *after_wrapper = (pe_action_wrapper_t *) gIter->data;
2548         /* pe_order_implies_then is the reason why a required A.start
2549          * implies/enforces B.start to be required too, which is the cause of
2550          * B.restart/re-promote.
2551          *
2552          * Not sure about pe_order_implies_then_on_node though. It's now only
2553          * used for unfencing case, which tends to introduce transition
2554          * loops...
2555          */
2556 
2557         if (!pcmk_is_set(after_wrapper->type, pe_order_implies_then)) {
2558             /* The order type between a group/clone and its child such as
2559              * B.start-> B_child.start is:
2560              * pe_order_implies_first_printed | pe_order_runnable_left
2561              *
2562              * Proceed through the ordering chain and build dependencies with
2563              * its children.
2564              */
2565             if (after->rsc == NULL
2566                 || after->rsc->variant < pe_group
2567                 || probe->rsc->parent == after->rsc
2568                 || after_wrapper->action->rsc == NULL
2569                 || after_wrapper->action->rsc->variant > pe_group
2570                 || after->rsc != after_wrapper->action->rsc->parent) {
2571                 continue;
2572             }
2573 
2574             /* Proceed to the children of a group or a non-interleaved clone.
2575              * For an interleaved clone, proceed only to the relevant child.
2576              */
2577             if (after->rsc->variant > pe_group
2578                 && interleave == TRUE
2579                 && (compatible_rsc == NULL
2580                     || compatible_rsc != after_wrapper->action->rsc)) {
2581                 continue;
2582             }
2583         }
2584 
2585         crm_trace("Proceeding through %s %s -> %s %s (type=0x%.6x)",
2586                   after->uuid,
2587                   after->node ? after->node->details->uname: "",
2588                   after_wrapper->action->uuid,
2589                   after_wrapper->action->node ? after_wrapper->action->node->details->uname : "",
2590                   after_wrapper->type);
2591 
2592         order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
2593     }
2594 }
2595 
2596 static void clear_actions_tracking_flag(pe_working_set_t * data_set)
     /*  */
2597 {
2598     GList *gIter = NULL;
2599 
2600     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2601         pe_action_t *action = (pe_action_t *) gIter->data;
2602 
2603         if (pcmk_is_set(action->flags, pe_action_tracking)) {
2604             pe__clear_action_flags(action, pe_action_tracking);
2605         }
2606     }
2607 }
2608 
2609 static void
2610 order_first_rsc_probes(pe_resource_t * rsc, pe_working_set_t * data_set)
     /*  */
2611 {
2612     GList *gIter = NULL;
2613     GList *probes = NULL;
2614 
2615     for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
2616         pe_resource_t * child = (pe_resource_t *) gIter->data;
2617 
2618         order_first_rsc_probes(child, data_set);
2619     }
2620 
2621     if (rsc->variant != pe_native) {
2622         return;
2623     }
2624 
2625     probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);
2626 
2627     for (gIter = probes; gIter != NULL; gIter= gIter->next) {
2628         pe_action_t *probe = (pe_action_t *) gIter->data;
2629         GList *aIter = NULL;
2630 
2631         for (aIter = probe->actions_after; aIter != NULL; aIter = aIter->next) {
2632             pe_action_wrapper_t *after_wrapper = (pe_action_wrapper_t *) aIter->data;
2633 
2634             order_first_probe_then_restart_repromote(probe, after_wrapper->action, data_set);
2635             clear_actions_tracking_flag(data_set);
2636         }
2637     }
2638 
2639     g_list_free(probes);
2640 }
2641 
2642 static void
2643 order_first_probes(pe_working_set_t * data_set)
     /*  */
2644 {
2645     GList *gIter = NULL;
2646 
2647     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2648         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
2649 
2650         order_first_rsc_probes(rsc, data_set);
2651     }
2652 
2653     order_first_probes_imply_stops(data_set);
2654 }
2655 
2656 static void
2657 order_then_probes(pe_working_set_t * data_set)
     /*  */
2658 {
2659 #if 0
2660     GList *gIter = NULL;
2661 
2662     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2663         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
2664 
2665         /* Given "A then B", we would prefer to wait for A to be
2666          * started before probing B.
2667          *
2668          * If A was a filesystem on which the binaries and data for B
2669          * lived, it would have been useful if the author of B's agent
2670          * could assume that A is running before B.monitor will be
2671          * called.
2672          *
2673          * However we can't _only_ probe once A is running, otherwise
2674          * we'd not detect the state of B if A could not be started
2675          * for some reason.
2676          *
2677          * In practice however, we cannot even do an opportunistic
2678          * version of this because B may be moving:
2679          *
2680          *   B.probe -> B.start
2681          *   B.probe -> B.stop
2682          *   B.stop -> B.start
2683          *   A.stop -> A.start
2684          *   A.start -> B.probe
2685          *
2686          * So far so good, but if we add the result of this code:
2687          *
2688          *   B.stop -> A.stop
2689          *
2690          * Then we get a loop:
2691          *
2692          *   B.probe -> B.stop -> A.stop -> A.start -> B.probe
2693          *
2694          * We could kill the 'B.probe -> B.stop' dependency, but that
2695          * could mean stopping B "too" soon, because B.start must wait
2696          * for the probes to complete.
2697          *
2698          * Another option is to allow it only if A is a non-unique
2699          * clone with clone-max == node-max (since we'll never be
2700          * moving it).  However, we could still be stopping one
2701          * instance at the same time as starting another.
2702 
2703          * The complexity of checking for allowed conditions combined
2704          * with the ever narrowing usecase suggests that this code
2705          * should remain disabled until someone gets smarter.
2706          */
2707         pe_action_t *start = NULL;
2708         GList *actions = NULL;
2709         GList *probes = NULL;
2710 
2711         actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);
2712 
2713         if (actions) {
2714             start = actions->data;
2715             g_list_free(actions);
2716         }
2717 
2718         if(start == NULL) {
2719             crm_err("No start action for %s", rsc->id);
2720             continue;
2721         }
2722 
2723         probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);
2724 
2725         for (actions = start->actions_before; actions != NULL; actions = actions->next) {
2726             pe_action_wrapper_t *before = (pe_action_wrapper_t *) actions->data;
2727 
2728             GList *pIter = NULL;
2729             pe_action_t *first = before->action;
2730             pe_resource_t *first_rsc = first->rsc;
2731 
2732             if(first->required_runnable_before) {
2733                 GList *clone_actions = NULL;
2734                 for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
2735                     before = (pe_action_wrapper_t *) clone_actions->data;
2736 
2737                     crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
2738 
2739                     CRM_ASSERT(before->action->rsc);
2740                     first_rsc = before->action->rsc;
2741                     break;
2742                 }
2743 
2744             } else if(!pcmk__str_eq(first->task, RSC_START, pcmk__str_casei)) {
2745                 crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
2746             }
2747 
2748             if(first_rsc == NULL) {
2749                 continue;
2750 
2751             } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
2752                 crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
2753                 continue;
2754 
2755             } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
2756                 crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
2757                 continue;
2758             }
2759 
2760             crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
2761 
2762             for (pIter = probes; pIter != NULL; pIter = pIter->next) {
2763                 pe_action_t *probe = (pe_action_t *) pIter->data;
2764 
2765                 crm_err("Ordering %s before %s", first->uuid, probe->uuid);
2766                 order_actions(first, probe, pe_order_optional);
2767             }
2768         }
2769     }
2770 #endif
2771 }
2772 
2773 static void
2774 order_probes(pe_working_set_t * data_set)
     /*  */
2775 {
2776     order_first_probes(data_set);
2777     order_then_probes(data_set);
2778 }
2779 
2780 gboolean
2781 stage7(pe_working_set_t * data_set)
     /*  */
2782 {
2783     pcmk__output_t *prev_out = data_set->priv;
2784     pcmk__output_t *out = NULL;
2785     GList *gIter = NULL;
2786 
2787     crm_trace("Applying ordering constraints");
2788 
2789     /* Don't ask me why, but apparently they need to be processed in
2790      * the order they were created in... go figure
2791      *
2792      * Also g_list_append() has horrendous performance characteristics
2793      * So we need to use g_list_prepend() and then reverse the list here
2794      */
2795     data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
2796 
2797     for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
2798         pe__ordering_t *order = gIter->data;
2799         pe_resource_t *rsc = order->lh_rsc;
2800 
2801         crm_trace("Applying ordering constraint: %d", order->id);
2802 
2803         if (rsc != NULL) {
2804             crm_trace("rsc_action-to-*");
2805             rsc_order_first(rsc, order, data_set);
2806             continue;
2807         }
2808 
2809         rsc = order->rh_rsc;
2810         if (rsc != NULL) {
2811             crm_trace("action-to-rsc_action");
2812             rsc_order_then(order->lh_action, rsc, order);
2813 
2814         } else {
2815             crm_trace("action-to-action");
2816             order_actions(order->lh_action, order->rh_action, order->type);
2817         }
2818     }
2819 
2820     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2821         pe_action_t *action = (pe_action_t *) gIter->data;
2822 
2823         update_colo_start_chain(action, data_set);
2824     }
2825 
2826     crm_trace("Ordering probes");
2827     order_probes(data_set);
2828 
2829     crm_trace("Updating %d actions", g_list_length(data_set->actions));
2830     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2831         pe_action_t *action = (pe_action_t *) gIter->data;
2832 
2833         update_action(action, data_set);
2834     }
2835 
2836     // Check for invalid orderings
2837     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2838         pe_action_t *action = (pe_action_t *) gIter->data;
2839         pe_action_wrapper_t *input = NULL;
2840 
2841         for (GList *input_iter = action->actions_before;
2842              input_iter != NULL; input_iter = input_iter->next) {
2843 
2844             input = (pe_action_wrapper_t *) input_iter->data;
2845             if (pcmk__ordering_is_invalid(action, input)) {
2846                 input->type = pe_order_none;
2847             }
2848         }
2849     }
2850 
2851     /* stage7 only ever outputs to the log, so ignore whatever output object was
2852      * previously set and just log instead.
2853      */
2854     out = pcmk__new_logger();
2855     if (out == NULL) {
2856         return FALSE;
2857     }
2858 
2859     pcmk__output_set_log_level(out, LOG_NOTICE);
2860     data_set->priv = out;
2861 
2862     out->begin_list(out, NULL, NULL, "Actions");
2863     LogNodeActions(data_set);
2864 
2865     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2866         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
2867 
2868         LogActions(rsc, data_set);
2869     }
2870 
2871     out->end_list(out);
2872     out->finish(out, CRM_EX_OK, true, NULL);
2873     pcmk__output_free(out);
2874 
2875     data_set->priv = prev_out;
2876     return TRUE;
2877 }
2878 
2879 static int transition_id = -1;
2880 
2881 /*!
2882  * \internal
2883  * \brief Log a message after calculating a transition
2884  *
2885  * \param[in] filename  Where transition input is stored
2886  */
2887 void
2888 pcmk__log_transition_summary(const char *filename)
     /*  */
2889 {
2890     if (was_processing_error) {
2891         crm_err("Calculated transition %d (with errors)%s%s",
2892                 transition_id,
2893                 (filename == NULL)? "" : ", saving inputs in ",
2894                 (filename == NULL)? "" : filename);
2895 
2896     } else if (was_processing_warning) {
2897         crm_warn("Calculated transition %d (with warnings)%s%s",
2898                  transition_id,
2899                  (filename == NULL)? "" : ", saving inputs in ",
2900                  (filename == NULL)? "" : filename);
2901 
2902     } else {
2903         crm_notice("Calculated transition %d%s%s",
2904                    transition_id,
2905                    (filename == NULL)? "" : ", saving inputs in ",
2906                    (filename == NULL)? "" : filename);
2907     }
2908     if (crm_config_error) {
2909         crm_notice("Configuration errors found during scheduler processing,"
2910                    "  please run \"crm_verify -L\" to identify issues");
2911     }
2912 }
2913 
2914 /*
2915  * Create a dependency graph to send to the transitioner (via the controller)
2916  */
2917 gboolean
2918 stage8(pe_working_set_t * data_set)
     /*  */
2919 {
2920     GList *gIter = NULL;
2921     const char *value = NULL;
2922     long long limit = 0LL;
2923 
2924     transition_id++;
2925     crm_trace("Creating transition graph %d.", transition_id);
2926 
2927     data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
2928 
2929     value = pe_pref(data_set->config_hash, "cluster-delay");
2930     crm_xml_add(data_set->graph, "cluster-delay", value);
2931 
2932     value = pe_pref(data_set->config_hash, "stonith-timeout");
2933     crm_xml_add(data_set->graph, "stonith-timeout", value);
2934 
2935     crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
2936 
2937     if (pcmk_is_set(data_set->flags, pe_flag_start_failure_fatal)) {
2938         crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
2939     } else {
2940         crm_xml_add(data_set->graph, "failed-start-offset", "1");
2941     }
2942 
2943     value = pe_pref(data_set->config_hash, "batch-limit");
2944     crm_xml_add(data_set->graph, "batch-limit", value);
2945 
2946     crm_xml_add_int(data_set->graph, "transition_id", transition_id);
2947 
2948     value = pe_pref(data_set->config_hash, "migration-limit");
2949     if ((pcmk__scan_ll(value, &limit, 0LL) == pcmk_rc_ok) && (limit > 0)) {
2950         crm_xml_add(data_set->graph, "migration-limit", value);
2951     }
2952 
2953     if (data_set->recheck_by > 0) {
2954         char *recheck_epoch = NULL;
2955 
2956         recheck_epoch = crm_strdup_printf("%llu",
2957                                           (long long) data_set->recheck_by);
2958         crm_xml_add(data_set->graph, "recheck-by", recheck_epoch);
2959         free(recheck_epoch);
2960     }
2961 
2962 /* errors...
2963    slist_iter(action, pe_action_t, action_list, lpc,
2964    if(action->optional == FALSE && action->runnable == FALSE) {
2965    print_action("Ignoring", action, TRUE);
2966    }
2967    );
2968 */
2969 
2970     /* The following code will de-duplicate action inputs, so nothing past this
2971      * should rely on the action input type flags retaining their original
2972      * values.
2973      */
2974 
2975     gIter = data_set->resources;
2976     for (; gIter != NULL; gIter = gIter->next) {
2977         pe_resource_t *rsc = (pe_resource_t *) gIter->data;
2978 
2979         pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
2980         rsc->cmds->expand(rsc, data_set);
2981     }
2982 
2983     crm_log_xml_trace(data_set->graph, "created resource-driven action list");
2984 
2985     /* pseudo action to distribute list of nodes with maintenance state update */
2986     add_maintenance_update(data_set);
2987 
2988     /* catch any non-resource specific actions */
2989     crm_trace("processing non-resource actions");
2990 
2991     gIter = data_set->actions;
2992     for (; gIter != NULL; gIter = gIter->next) {
2993         pe_action_t *action = (pe_action_t *) gIter->data;
2994 
2995         if (action->rsc
2996             && action->node
2997             && action->node->details->shutdown
2998             && !pcmk_is_set(action->rsc->flags, pe_rsc_maintenance)
2999             && !pcmk_any_flags_set(action->flags,
3000                                    pe_action_optional|pe_action_runnable)
3001             && pcmk__str_eq(action->task, RSC_STOP, pcmk__str_none)
3002             ) {
3003             /* Eventually we should just ignore the 'fence' case
3004              * But for now it's the best way to detect (in CTS) when
3005              * CIB resource updates are being lost
3006              */
3007             if (pcmk_is_set(data_set->flags, pe_flag_have_quorum)
3008                 || data_set->no_quorum_policy == no_quorum_ignore) {
3009                 crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
3010                          action->node->details->unclean ? "fence" : "shut down",
3011                          action->node->details->uname, action->rsc->id,
3012                          pcmk_is_set(action->rsc->flags, pe_rsc_managed)? " blocked" : " unmanaged",
3013                          pcmk_is_set(action->rsc->flags, pe_rsc_failed)? " failed" : "",
3014                          action->uuid);
3015             }
3016         }
3017 
3018         graph_element_from_action(action, data_set);
3019     }
3020 
3021     crm_log_xml_trace(data_set->graph, "created generic action list");
3022     crm_trace("Created transition graph %d.", transition_id);
3023 
3024     return TRUE;
3025 }
3026 
3027 void
3028 LogNodeActions(pe_working_set_t * data_set)
     /*  */
3029 {
3030     pcmk__output_t *out = data_set->priv;
3031     GList *gIter = NULL;
3032 
3033     for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
3034         char *node_name = NULL;
3035         char *task = NULL;
3036         pe_action_t *action = (pe_action_t *) gIter->data;
3037 
3038         if (action->rsc != NULL) {
3039             continue;
3040         } else if (pcmk_is_set(action->flags, pe_action_optional)) {
3041             continue;
3042         }
3043 
3044         if (pe__is_guest_node(action->node)) {
3045             node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
3046         } else if(action->node) {
3047             node_name = crm_strdup_printf("%s", action->node->details->uname);
3048         }
3049 
3050 
3051         if (pcmk__str_eq(action->task, CRM_OP_SHUTDOWN, pcmk__str_casei)) {
3052             task = strdup("Shutdown");
3053         } else if (pcmk__str_eq(action->task, CRM_OP_FENCE, pcmk__str_casei)) {
3054             const char *op = g_hash_table_lookup(action->meta, "stonith_action");
3055             task = crm_strdup_printf("Fence (%s)", op);
3056         }
3057 
3058         out->message(out, "node-action", task, node_name, action->reason);
3059 
3060         free(node_name);
3061         free(task);
3062     }
3063 }
/* */
root/lib/pacemaker/pcmk_sched_allocate.c

DEFINITIONS
