pacemaker  2.0.2-debe490
Scalable High-Availability cluster resource manager
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
pcmk_sched_allocate.c
Go to the documentation of this file.
1 /*
2  * Copyright 2004-2019 the Pacemaker project contributors
3  *
4  * The version control history for this file may have further details.
5  *
6  * This source code is licensed under the GNU General Public License version 2
7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
8  */
9 
10 #include <crm_internal.h>
11 
12 #include <sys/param.h>
13 
14 #include <crm/crm.h>
15 #include <crm/cib.h>
16 #include <crm/msg_xml.h>
17 #include <crm/common/xml.h>
18 
19 #include <glib.h>
20 
21 #include <crm/pengine/status.h>
22 #include <pacemaker-internal.h>
23 
24 CRM_TRACE_INIT_DATA(pe_allocate);
25 
26 void set_alloc_actions(pe_working_set_t * data_set);
27 extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set);
28 extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set);
29 static void apply_remote_node_ordering(pe_working_set_t *data_set);
30 static enum remote_connection_state get_remote_node_state(pe_node_t *node);
31 
38 };
39 
40 static const char *
41 state2text(enum remote_connection_state state)
42 {
43  switch (state) {
45  return "unknown";
46  case remote_state_alive:
47  return "alive";
49  return "resting";
51  return "failed";
53  return "stopped";
54  }
55 
56  return "impossible";
57 }
58 
60  {
73  },
74  {
87  },
88  {
101  },
102  {
115  }
116 };
117 
118 gboolean
119 update_action_flags(action_t * action, enum pe_action_flags flags, const char *source, int line)
120 {
121  static unsigned long calls = 0;
122  gboolean changed = FALSE;
123  gboolean clear = is_set(flags, pe_action_clear);
124  enum pe_action_flags last = action->flags;
125 
126  if (clear) {
127  action->flags = crm_clear_bit(source, line, action->uuid, action->flags, flags);
128  } else {
129  action->flags = crm_set_bit(source, line, action->uuid, action->flags, flags);
130  }
131 
132  if (last != action->flags) {
133  calls++;
134  changed = TRUE;
135  /* Useful for tracking down _who_ changed a specific flag */
136  /* CRM_ASSERT(calls != 534); */
137  clear_bit(flags, pe_action_clear);
138  crm_trace("%s on %s: %sset flags 0x%.6x (was 0x%.6x, now 0x%.6x, %lu, %s)",
139  action->uuid, action->node ? action->node->details->uname : "[none]",
140  clear ? "un-" : "", flags, last, action->flags, calls, source);
141  }
142 
143  return changed;
144 }
145 
146 static gboolean
147 check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry,
148  gboolean active_here, pe_working_set_t * data_set)
149 {
150  int attr_lpc = 0;
151  gboolean force_restart = FALSE;
152  gboolean delete_resource = FALSE;
153  gboolean changed = FALSE;
154 
155  const char *value = NULL;
156  const char *old_value = NULL;
157 
158  const char *attr_list[] = {
162  };
163 
164  for (; attr_lpc < DIMOF(attr_list); attr_lpc++) {
165  value = crm_element_value(rsc->xml, attr_list[attr_lpc]);
166  old_value = crm_element_value(rsc_entry, attr_list[attr_lpc]);
167  if (value == old_value /* i.e. NULL */
168  || crm_str_eq(value, old_value, TRUE)) {
169  continue;
170  }
171 
172  changed = TRUE;
173  trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set);
174  if (active_here) {
175  force_restart = TRUE;
176  crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s",
177  rsc->id, node->details->uname, attr_list[attr_lpc],
178  crm_str(old_value), crm_str(value));
179  }
180  }
181  if (force_restart) {
182  /* make sure the restart happens */
183  stop_action(rsc, node, FALSE);
185  delete_resource = TRUE;
186 
187  } else if (changed) {
188  delete_resource = TRUE;
189  }
190  return delete_resource;
191 }
192 
193 static void
194 CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node,
195  const char *reason, pe_working_set_t * data_set)
196 {
197  guint interval_ms = 0;
198  action_t *cancel = NULL;
199 
200  const char *task = NULL;
201  const char *call_id = NULL;
202  const char *interval_ms_s = NULL;
203 
204  CRM_CHECK(xml_op != NULL, return);
205  CRM_CHECK(active_node != NULL, return);
206 
207  task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
208  call_id = crm_element_value(xml_op, XML_LRM_ATTR_CALLID);
209  interval_ms_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
210 
211  interval_ms = crm_parse_ms(interval_ms_s);
212 
213  crm_info("Action " CRM_OP_FMT " on %s will be stopped: %s",
214  rsc->id, task, interval_ms,
215  active_node->details->uname, (reason? reason : "unknown"));
216 
217  cancel = pe_cancel_op(rsc, task, interval_ms, active_node, data_set);
218  add_hash_param(cancel->meta, XML_LRM_ATTR_CALLID, call_id);
219  custom_action_order(rsc, stop_key(rsc), NULL, rsc, NULL, cancel, pe_order_optional, data_set);
220 }
221 
222 static gboolean
223 check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op,
224  pe_working_set_t * data_set)
225 {
226  char *key = NULL;
227  guint interval_ms = 0;
228  const char *interval_ms_s = NULL;
229  const op_digest_cache_t *digest_data = NULL;
230  gboolean did_change = FALSE;
231 
232  const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK);
233  const char *digest_secure = NULL;
234 
235  CRM_CHECK(active_node != NULL, return FALSE);
236 
237  interval_ms_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL_MS);
238  interval_ms = crm_parse_ms(interval_ms_s);
239 
240  if (interval_ms > 0) {
241  xmlNode *op_match = NULL;
242 
243  /* we need to reconstruct the key because of the way we used to construct resource IDs */
244  key = generate_op_key(rsc->id, task, interval_ms);
245 
246  pe_rsc_trace(rsc, "Checking parameters for %s", key);
247  op_match = find_rsc_op_entry(rsc, key);
248 
249  if (op_match == NULL && is_set(data_set->flags, pe_flag_stop_action_orphans)) {
250  CancelXmlOp(rsc, xml_op, active_node, "orphan", data_set);
251  free(key);
252  return TRUE;
253 
254  } else if (op_match == NULL) {
255  pe_rsc_debug(rsc, "Orphan action detected: %s on %s", key, active_node->details->uname);
256  free(key);
257  return TRUE;
258  }
259  free(key);
260  key = NULL;
261  }
262 
263  crm_trace("Testing " CRM_OP_FMT " on %s",
264  rsc->id, task, interval_ms, active_node->details->uname);
265  if ((interval_ms == 0) && safe_str_eq(task, RSC_STATUS)) {
266  /* Reload based on the start action not a probe */
267  task = RSC_START;
268 
269  } else if ((interval_ms == 0) && safe_str_eq(task, RSC_MIGRATED)) {
270  /* Reload based on the start action not a migrate */
271  task = RSC_START;
272  } else if ((interval_ms == 0) && safe_str_eq(task, RSC_PROMOTE)) {
273  /* Reload based on the start action not a promote */
274  task = RSC_START;
275  }
276 
277  digest_data = rsc_action_digest_cmp(rsc, xml_op, active_node, data_set);
278 
279  if(is_set(data_set->flags, pe_flag_sanitized)) {
280  digest_secure = crm_element_value(xml_op, XML_LRM_ATTR_SECURE_DIGEST);
281  }
282 
283  if(digest_data->rc != RSC_DIGEST_MATCH
284  && digest_secure
285  && digest_data->digest_secure_calc
286  && strcmp(digest_data->digest_secure_calc, digest_secure) == 0) {
287  if (is_set(data_set->flags, pe_flag_stdout)) {
288  printf("Only 'private' parameters to " CRM_OP_FMT " on %s changed: %s\n",
289  rsc->id, task, interval_ms, active_node->details->uname,
291  }
292 
293  } else if (digest_data->rc == RSC_DIGEST_RESTART) {
294  /* Changes that force a restart */
295  pe_action_t *required = NULL;
296 
297  did_change = TRUE;
298  key = generate_op_key(rsc->id, task, interval_ms);
299  crm_log_xml_info(digest_data->params_restart, "params:restart");
300  required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
301  pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
302  "resource definition change", pe_action_optional, TRUE);
303 
304  trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set);
305 
306  } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) {
307  /* Changes that can potentially be handled by a reload */
308  const char *digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST);
309 
310  did_change = TRUE;
311  trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set);
312  crm_log_xml_info(digest_data->params_all, "params:reload");
313  key = generate_op_key(rsc->id, task, interval_ms);
314 
315  if (interval_ms > 0) {
316  action_t *op = NULL;
317 
318 #if 0
319  /* Always reload/restart the entire resource */
320  ReloadRsc(rsc, active_node, data_set);
321 #else
322  /* Re-sending the recurring op is sufficient - the old one will be cancelled automatically */
323  op = custom_action(rsc, key, task, active_node, TRUE, TRUE, data_set);
325 #endif
326 
327  } else if (digest_restart) {
328  pe_rsc_trace(rsc, "Reloading '%s' action for resource %s", task, rsc->id);
329 
330  /* Reload this resource */
331  ReloadRsc(rsc, active_node, data_set);
332  free(key);
333 
334  } else {
335  pe_action_t *required = NULL;
336  pe_rsc_trace(rsc, "Resource %s doesn't know how to reload", rsc->id);
337 
338  /* Re-send the start/demote/promote op
339  * Recurring ops will be detected independently
340  */
341  required = custom_action(rsc, key, task, NULL, TRUE, TRUE, data_set);
342  pe_action_set_flag_reason(__FUNCTION__, __LINE__, required, NULL,
343  "resource definition change", pe_action_optional, TRUE);
344  }
345  }
346 
347  return did_change;
348 }
349 
356 static void
357 check_params(pe_resource_t *rsc, pe_node_t *node, xmlNode *rsc_op,
358  enum pe_check_parameters check, pe_working_set_t *data_set)
359 {
360  const char *reason = NULL;
361  op_digest_cache_t *digest_data = NULL;
362 
363  switch (check) {
364  case pe_check_active:
365  if (check_action_definition(rsc, node, rsc_op, data_set)
366  && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
367  data_set)) {
368 
369  reason = "action definition changed";
370  }
371  break;
372 
374  digest_data = rsc_action_digest_cmp(rsc, rsc_op, node, data_set);
375  switch (digest_data->rc) {
376  case RSC_DIGEST_UNKNOWN:
377  crm_trace("Resource %s history entry %s on %s has no digest to compare",
378  rsc->id, ID(rsc_op), node->details->id);
379  break;
380  case RSC_DIGEST_MATCH:
381  break;
382  default:
383  reason = "resource parameters have changed";
384  break;
385  }
386  break;
387  }
388 
389  if (reason) {
390  pe__clear_failcount(rsc, node, reason, data_set);
391  }
392 }
393 
394 static void
395 check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_working_set_t * data_set)
396 {
397  GListPtr gIter = NULL;
398  int offset = -1;
399  guint interval_ms = 0;
400  int stop_index = 0;
401  int start_index = 0;
402 
403  const char *task = NULL;
404  const char *interval_ms_s = NULL;
405 
406  xmlNode *rsc_op = NULL;
407  GListPtr op_list = NULL;
408  GListPtr sorted_op_list = NULL;
409 
410  CRM_CHECK(node != NULL, return);
411 
412  if (is_set(rsc->flags, pe_rsc_orphan)) {
413  resource_t *parent = uber_parent(rsc);
414  if(parent == NULL
415  || pe_rsc_is_clone(parent) == FALSE
416  || is_set(parent->flags, pe_rsc_unique)) {
417  pe_rsc_trace(rsc, "Skipping param check for %s and deleting: orphan", rsc->id);
418  DeleteRsc(rsc, node, FALSE, data_set);
419  } else {
420  pe_rsc_trace(rsc, "Skipping param check for %s (orphan clone)", rsc->id);
421  }
422  return;
423 
424  } else if (pe_find_node_id(rsc->running_on, node->details->id) == NULL) {
425  if (check_rsc_parameters(rsc, node, rsc_entry, FALSE, data_set)) {
426  DeleteRsc(rsc, node, FALSE, data_set);
427  }
428  pe_rsc_trace(rsc, "Skipping param check for %s: no longer active on %s",
429  rsc->id, node->details->uname);
430  return;
431  }
432 
433  pe_rsc_trace(rsc, "Processing %s on %s", rsc->id, node->details->uname);
434 
435  if (check_rsc_parameters(rsc, node, rsc_entry, TRUE, data_set)) {
436  DeleteRsc(rsc, node, FALSE, data_set);
437  }
438 
439  for (rsc_op = __xml_first_child(rsc_entry); rsc_op != NULL; rsc_op = __xml_next_element(rsc_op)) {
440  if (crm_str_eq((const char *)rsc_op->name, XML_LRM_TAG_RSC_OP, TRUE)) {
441  op_list = g_list_prepend(op_list, rsc_op);
442  }
443  }
444 
445  sorted_op_list = g_list_sort(op_list, sort_op_by_callid);
446  calculate_active_ops(sorted_op_list, &start_index, &stop_index);
447 
448  for (gIter = sorted_op_list; gIter != NULL; gIter = gIter->next) {
449  xmlNode *rsc_op = (xmlNode *) gIter->data;
450 
451  offset++;
452 
453  if (start_index < stop_index) {
454  /* stopped */
455  continue;
456  } else if (offset < start_index) {
457  /* action occurred prior to a start */
458  continue;
459  }
460 
461  task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
462 
463  interval_ms_s = crm_element_value(rsc_op, XML_LRM_ATTR_INTERVAL_MS);
464  interval_ms = crm_parse_ms(interval_ms_s);
465 
466  if ((interval_ms > 0) &&
467  (is_set(rsc->flags, pe_rsc_maintenance) || node->details->maintenance)) {
468  // Maintenance mode cancels recurring operations
469  CancelXmlOp(rsc, rsc_op, node, "maintenance mode", data_set);
470 
471  } else if ((interval_ms > 0)
472  || safe_str_eq(task, RSC_STATUS)
473  || safe_str_eq(task, RSC_START)
474  || safe_str_eq(task, RSC_PROMOTE)
475  || safe_str_eq(task, RSC_MIGRATED)) {
476 
477  /* If a resource operation failed, and the operation's definition
478  * has changed, clear any fail count so they can be retried fresh.
479  */
480 
481  if (pe__bundle_needs_remote_name(rsc)) {
482  /* We haven't allocated resources to nodes yet, so if the
483  * REMOTE_CONTAINER_HACK is used, we may calculate the digest
484  * based on the literal "#uname" value rather than the properly
485  * substituted value. That would mistakenly make the action
486  * definition appear to have been changed. Defer the check until
487  * later in this case.
488  */
489  pe__add_param_check(rsc_op, rsc, node, pe_check_active,
490  data_set);
491 
492  } else if (check_action_definition(rsc, node, rsc_op, data_set)
493  && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
494  data_set)) {
495  pe__clear_failcount(rsc, node, "action definition changed",
496  data_set);
497  }
498  }
499  }
500  g_list_free(sorted_op_list);
501 }
502 
503 static GListPtr
504 find_rsc_list(GListPtr result, resource_t * rsc, const char *id, gboolean renamed_clones,
505  gboolean partial, pe_working_set_t * data_set)
506 {
507  GListPtr gIter = NULL;
508  gboolean match = FALSE;
509 
510  if (id == NULL) {
511  return NULL;
512 
513  } else if (rsc == NULL && data_set) {
514 
515  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
516  resource_t *child = (resource_t *) gIter->data;
517 
518  result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
519  }
520 
521  return result;
522 
523  } else if (rsc == NULL) {
524  return NULL;
525  }
526 
527  if (partial) {
528  if (strstr(rsc->id, id)) {
529  match = TRUE;
530 
531  } else if (renamed_clones && rsc->clone_name && strstr(rsc->clone_name, id)) {
532  match = TRUE;
533  }
534 
535  } else {
536  if (strcmp(rsc->id, id) == 0) {
537  match = TRUE;
538 
539  } else if (renamed_clones && rsc->clone_name && strcmp(rsc->clone_name, id) == 0) {
540  match = TRUE;
541  }
542  }
543 
544  if (match) {
545  result = g_list_prepend(result, rsc);
546  }
547 
548  if (rsc->children) {
549  gIter = rsc->children;
550  for (; gIter != NULL; gIter = gIter->next) {
551  resource_t *child = (resource_t *) gIter->data;
552 
553  result = find_rsc_list(result, child, id, renamed_clones, partial, NULL);
554  }
555  }
556 
557  return result;
558 }
559 
560 static void
561 check_actions(pe_working_set_t * data_set)
562 {
563  const char *id = NULL;
564  node_t *node = NULL;
565  xmlNode *lrm_rscs = NULL;
566  xmlNode *status = get_object_root(XML_CIB_TAG_STATUS, data_set->input);
567 
568  xmlNode *node_state = NULL;
569 
570  for (node_state = __xml_first_child(status); node_state != NULL;
571  node_state = __xml_next_element(node_state)) {
572  if (crm_str_eq((const char *)node_state->name, XML_CIB_TAG_STATE, TRUE)) {
573  id = crm_element_value(node_state, XML_ATTR_ID);
574  lrm_rscs = find_xml_node(node_state, XML_CIB_TAG_LRM, FALSE);
575  lrm_rscs = find_xml_node(lrm_rscs, XML_LRM_TAG_RESOURCES, FALSE);
576 
577  node = pe_find_node_id(data_set->nodes, id);
578 
579  if (node == NULL) {
580  continue;
581 
582  /* Still need to check actions for a maintenance node to cancel existing monitor operations */
583  } else if (can_run_resources(node) == FALSE && node->details->maintenance == FALSE) {
584  crm_trace("Skipping param check for %s: can't run resources",
585  node->details->uname);
586  continue;
587  }
588 
589  crm_trace("Processing node %s", node->details->uname);
590  if (node->details->online || is_set(data_set->flags, pe_flag_stonith_enabled)) {
591  xmlNode *rsc_entry = NULL;
592 
593  for (rsc_entry = __xml_first_child(lrm_rscs); rsc_entry != NULL;
594  rsc_entry = __xml_next_element(rsc_entry)) {
595  if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) {
596 
597  if (xml_has_children(rsc_entry)) {
598  GListPtr gIter = NULL;
599  GListPtr result = NULL;
600  const char *rsc_id = ID(rsc_entry);
601 
602  CRM_CHECK(rsc_id != NULL, return);
603 
604  result = find_rsc_list(NULL, NULL, rsc_id, TRUE, FALSE, data_set);
605  for (gIter = result; gIter != NULL; gIter = gIter->next) {
606  resource_t *rsc = (resource_t *) gIter->data;
607 
608  if (rsc->variant != pe_native) {
609  continue;
610  }
611  check_actions_for(rsc_entry, rsc, node, data_set);
612  }
613  g_list_free(result);
614  }
615  }
616  }
617  }
618  }
619  }
620 }
621 
622 static gboolean
623 apply_placement_constraints(pe_working_set_t * data_set)
624 {
625  GListPtr gIter = NULL;
626 
627  crm_trace("Applying constraints...");
628 
629  for (gIter = data_set->placement_constraints; gIter != NULL; gIter = gIter->next) {
630  pe__location_t *cons = gIter->data;
631 
632  cons->rsc_lh->cmds->rsc_location(cons->rsc_lh, cons);
633  }
634 
635  return TRUE;
636 
637 }
638 
639 static gboolean
640 failcount_clear_action_exists(node_t * node, resource_t * rsc)
641 {
642  gboolean rc = FALSE;
643  GList *list = pe__resource_actions(rsc, node, CRM_OP_CLEAR_FAILCOUNT, TRUE);
644 
645  if (list) {
646  rc = TRUE;
647  }
648  g_list_free(list);
649  return rc;
650 }
651 
660 static void
661 check_migration_threshold(resource_t *rsc, node_t *node,
662  pe_working_set_t *data_set)
663 {
664  int fail_count, countdown;
665  resource_t *failed;
666 
667  /* Migration threshold of 0 means never force away */
668  if (rsc->migration_threshold == 0) {
669  return;
670  }
671 
672  // If we're ignoring failures, also ignore the migration threshold
673  if (is_set(rsc->flags, pe_rsc_failure_ignored)) {
674  return;
675  }
676 
677  /* If there are no failures, there's no need to force away */
678  fail_count = pe_get_failcount(node, rsc, NULL,
680  data_set);
681  if (fail_count <= 0) {
682  return;
683  }
684 
685  /* How many more times recovery will be tried on this node */
686  countdown = QB_MAX(rsc->migration_threshold - fail_count, 0);
687 
688  /* If failed resource has a parent, we'll force the parent away */
689  failed = rsc;
690  if (is_not_set(rsc->flags, pe_rsc_unique)) {
691  failed = uber_parent(rsc);
692  }
693 
694  if (countdown == 0) {
695  resource_location(failed, node, -INFINITY, "__fail_limit__", data_set);
696  crm_warn("Forcing %s away from %s after %d failures (max=%d)",
697  failed->id, node->details->uname, fail_count,
698  rsc->migration_threshold);
699  } else {
700  crm_info("%s can fail %d more times on %s before being forced off",
701  failed->id, countdown, node->details->uname);
702  }
703 }
704 
705 static void
706 common_apply_stickiness(resource_t * rsc, node_t * node, pe_working_set_t * data_set)
707 {
708  if (rsc->children) {
709  GListPtr gIter = rsc->children;
710 
711  for (; gIter != NULL; gIter = gIter->next) {
712  resource_t *child_rsc = (resource_t *) gIter->data;
713 
714  common_apply_stickiness(child_rsc, node, data_set);
715  }
716  return;
717  }
718 
719  if (is_set(rsc->flags, pe_rsc_managed)
720  && rsc->stickiness != 0 && g_list_length(rsc->running_on) == 1) {
721  node_t *current = pe_find_node_id(rsc->running_on, node->details->id);
722  node_t *match = pe_hash_table_lookup(rsc->allowed_nodes, node->details->id);
723 
724  if (current == NULL) {
725 
726  } else if (match != NULL || is_set(data_set->flags, pe_flag_symmetric_cluster)) {
727  resource_t *sticky_rsc = rsc;
728 
729  resource_location(sticky_rsc, node, rsc->stickiness, "stickiness", data_set);
730  pe_rsc_debug(sticky_rsc, "Resource %s: preferring current location"
731  " (node=%s, weight=%d)", sticky_rsc->id,
732  node->details->uname, rsc->stickiness);
733  } else {
734  GHashTableIter iter;
735  node_t *nIter = NULL;
736 
737  pe_rsc_debug(rsc, "Ignoring stickiness for %s: the cluster is asymmetric"
738  " and node %s is not explicitly allowed", rsc->id, node->details->uname);
739  g_hash_table_iter_init(&iter, rsc->allowed_nodes);
740  while (g_hash_table_iter_next(&iter, NULL, (void **)&nIter)) {
741  crm_err("%s[%s] = %d", rsc->id, nIter->details->uname, nIter->weight);
742  }
743  }
744  }
745 
746  /* Check the migration threshold only if a failcount clear action
747  * has not already been placed for this resource on the node.
748  * There is no sense in potentially forcing the resource from this
749  * node if the failcount is being reset anyway.
750  *
751  * @TODO A clear_failcount operation can be scheduled in stage4() via
752  * check_actions_for(), or in stage5() via check_params(). This runs in
753  * stage2(), so it cannot detect those, meaning we might check the migration
754  * threshold when we shouldn't -- worst case, we stop or move the resource,
755  * then move it back next transition.
756  */
757  if (failcount_clear_action_exists(node, rsc) == FALSE) {
758  check_migration_threshold(rsc, node, data_set);
759  }
760 }
761 
762 void
764 {
765  GListPtr gIter = rsc->children;
766 
767  rsc->cmds = &resource_class_alloc_functions[rsc->variant];
768 
769  for (; gIter != NULL; gIter = gIter->next) {
770  resource_t *child_rsc = (resource_t *) gIter->data;
771 
772  complex_set_cmds(child_rsc);
773  }
774 }
775 
776 void
778 {
779 
780  GListPtr gIter = data_set->resources;
781 
782  for (; gIter != NULL; gIter = gIter->next) {
783  resource_t *rsc = (resource_t *) gIter->data;
784 
785  complex_set_cmds(rsc);
786  }
787 }
788 
789 static void
790 calculate_system_health(gpointer gKey, gpointer gValue, gpointer user_data)
791 {
792  const char *key = (const char *)gKey;
793  const char *value = (const char *)gValue;
794  int *system_health = (int *)user_data;
795 
796  if (!gKey || !gValue || !user_data) {
797  return;
798  }
799 
800  if (crm_starts_with(key, "#health")) {
801  int score;
802 
803  /* Convert the value into an integer */
804  score = char2score(value);
805 
806  /* Add it to the running total */
807  *system_health = merge_weights(score, *system_health);
808  }
809 }
810 
811 static gboolean
812 apply_system_health(pe_working_set_t * data_set)
813 {
814  GListPtr gIter = NULL;
815  const char *health_strategy = pe_pref(data_set->config_hash, "node-health-strategy");
816  int base_health = 0;
817 
818  if (health_strategy == NULL || safe_str_eq(health_strategy, "none")) {
819  /* Prevent any accidental health -> score translation */
820  node_score_red = 0;
821  node_score_yellow = 0;
822  node_score_green = 0;
823  return TRUE;
824 
825  } else if (safe_str_eq(health_strategy, "migrate-on-red")) {
826 
827  /* Resources on nodes which have health values of red are
828  * weighted away from that node.
829  */
831  node_score_yellow = 0;
832  node_score_green = 0;
833 
834  } else if (safe_str_eq(health_strategy, "only-green")) {
835 
836  /* Resources on nodes which have health values of red or yellow
837  * are forced away from that node.
838  */
841  node_score_green = 0;
842 
843  } else if (safe_str_eq(health_strategy, "progressive")) {
844  /* Same as the above, but use the r/y/g scores provided by the user
845  * Defaults are provided by the pe_prefs table
846  * Also, custom health "base score" can be used
847  */
848  base_health = crm_parse_int(pe_pref(data_set->config_hash, "node-health-base"), "0");
849 
850  } else if (safe_str_eq(health_strategy, "custom")) {
851 
852  /* Requires the admin to configure the rsc_location constaints for
853  * processing the stored health scores
854  */
855  /* TODO: Check for the existence of appropriate node health constraints */
856  return TRUE;
857 
858  } else {
859  crm_err("Unknown node health strategy: %s", health_strategy);
860  return FALSE;
861  }
862 
863  crm_info("Applying automated node health strategy: %s", health_strategy);
864 
865  for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
866  int system_health = base_health;
867  node_t *node = (node_t *) gIter->data;
868 
869  /* Search through the node hash table for system health entries. */
870  g_hash_table_foreach(node->details->attrs, calculate_system_health, &system_health);
871 
872  crm_info(" Node %s has an combined system health of %d",
873  node->details->uname, system_health);
874 
875  /* If the health is non-zero, then create a new rsc2node so that the
876  * weight will be added later on.
877  */
878  if (system_health != 0) {
879 
880  GListPtr gIter2 = data_set->resources;
881 
882  for (; gIter2 != NULL; gIter2 = gIter2->next) {
883  resource_t *rsc = (resource_t *) gIter2->data;
884 
885  rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set);
886  }
887  }
888  }
889 
890  return TRUE;
891 }
892 
893 gboolean
895 {
896  xmlNode *cib_constraints = get_object_root(XML_CIB_TAG_CONSTRAINTS, data_set->input);
897 
898  if (data_set->input == NULL) {
899  return FALSE;
900  }
901 
902  if (is_set(data_set->flags, pe_flag_have_status) == FALSE) {
903  crm_trace("Calculating status");
904  cluster_status(data_set);
905  }
906 
907  set_alloc_actions(data_set);
908  apply_system_health(data_set);
909  unpack_constraints(cib_constraints, data_set);
910 
911  return TRUE;
912 }
913 
914 /*
915  * Check nodes for resources started outside of the LRM
916  */
917 gboolean
919 {
920  action_t *probe_node_complete = NULL;
921 
922  for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
923  node_t *node = (node_t *) gIter->data;
924  const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED);
925 
926  if (node->details->online == FALSE) {
927 
928  if (pe__is_remote_node(node) && node->details->remote_rsc
929  && (get_remote_node_state(node) == remote_state_failed)) {
930 
931  pe_fence_node(data_set, node, "the connection is unrecoverable");
932  }
933  continue;
934 
935  } else if (node->details->unclean) {
936  continue;
937 
938  } else if (node->details->rsc_discovery_enabled == FALSE) {
939  /* resource discovery is disabled for this node */
940  continue;
941  }
942 
943  if (probed != NULL && crm_is_true(probed) == FALSE) {
944  action_t *probe_op = custom_action(NULL, crm_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname),
945  CRM_OP_REPROBE, node, FALSE, TRUE, data_set);
946 
948  continue;
949  }
950 
951  for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) {
952  resource_t *rsc = (resource_t *) gIter2->data;
953 
954  rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set);
955  }
956  }
957  return TRUE;
958 }
959 
960 static void
961 rsc_discover_filter(resource_t *rsc, node_t *node)
962 {
963  GListPtr gIter = rsc->children;
964  resource_t *top = uber_parent(rsc);
965  node_t *match;
966 
967  if (rsc->exclusive_discover == FALSE && top->exclusive_discover == FALSE) {
968  return;
969  }
970 
971  for (; gIter != NULL; gIter = gIter->next) {
972  resource_t *child_rsc = (resource_t *) gIter->data;
973  rsc_discover_filter(child_rsc, node);
974  }
975 
976  match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id);
977  if (match && match->rsc_discover_mode != pe_discover_exclusive) {
978  match->weight = -INFINITY;
979  }
980 }
981 
982 /*
983  * Count how many valid nodes we have (so we know the maximum number of
984  * colors we can resolve).
985  *
986  * Apply node constraints (i.e. filter the "allowed_nodes" part of resources)
987  */
988 gboolean
990 {
991  GListPtr gIter = NULL;
992 
993  crm_trace("Applying placement constraints");
994 
995  gIter = data_set->nodes;
996  for (; gIter != NULL; gIter = gIter->next) {
997  node_t *node = (node_t *) gIter->data;
998 
999  if (node == NULL) {
1000  /* error */
1001 
1002  } else if (node->weight >= 0.0 /* global weight */
1003  && node->details->online && node->details->type != node_ping) {
1004  data_set->max_valid_nodes++;
1005  }
1006  }
1007 
1008  apply_placement_constraints(data_set);
1009 
1010  gIter = data_set->nodes;
1011  for (; gIter != NULL; gIter = gIter->next) {
1012  GListPtr gIter2 = NULL;
1013  node_t *node = (node_t *) gIter->data;
1014 
1015  gIter2 = data_set->resources;
1016  for (; gIter2 != NULL; gIter2 = gIter2->next) {
1017  resource_t *rsc = (resource_t *) gIter2->data;
1018 
1019  common_apply_stickiness(rsc, node, data_set);
1020  rsc_discover_filter(rsc, node);
1021  }
1022  }
1023 
1024  return TRUE;
1025 }
1026 
1027 /*
1028  * Create internal resource constraints before allocation
1029  */
1030 gboolean
1032 {
1033 
1034  GListPtr gIter = data_set->resources;
1035 
1036  for (; gIter != NULL; gIter = gIter->next) {
1037  resource_t *rsc = (resource_t *) gIter->data;
1038 
1039  rsc->cmds->internal_constraints(rsc, data_set);
1040  }
1041 
1042  return TRUE;
1043 }
1044 
1045 /*
1046  * Check for orphaned or redefined actions
1047  */
1048 gboolean
1050 {
1051  check_actions(data_set);
1052  return TRUE;
1053 }
1054 
1055 static void *
1056 convert_const_pointer(const void *ptr)
1057 {
1058  /* Worst function ever */
1059  return (void *)ptr;
1060 }
1061 
1062 static gint
1063 sort_rsc_process_order(gconstpointer a, gconstpointer b, gpointer data)
1064 {
1065  int rc = 0;
1066  int r1_weight = -INFINITY;
1067  int r2_weight = -INFINITY;
1068 
1069  const char *reason = "existence";
1070 
1071  const GListPtr nodes = (GListPtr) data;
1072  const resource_t *resource1 = a;
1073  const resource_t *resource2 = b;
1074 
1075  node_t *r1_node = NULL;
1076  node_t *r2_node = NULL;
1077  GListPtr gIter = NULL;
1078  GHashTable *r1_nodes = NULL;
1079  GHashTable *r2_nodes = NULL;
1080 
1081  if (a == NULL && b == NULL) {
1082  goto done;
1083  }
1084  if (a == NULL) {
1085  return 1;
1086  }
1087  if (b == NULL) {
1088  return -1;
1089  }
1090 
1091  reason = "priority";
1092  r1_weight = resource1->priority;
1093  r2_weight = resource2->priority;
1094 
1095  if (r1_weight > r2_weight) {
1096  rc = -1;
1097  goto done;
1098  }
1099 
1100  if (r1_weight < r2_weight) {
1101  rc = 1;
1102  goto done;
1103  }
1104 
1105  reason = "no node list";
1106  if (nodes == NULL) {
1107  goto done;
1108  }
1109 
1110  r1_nodes = rsc_merge_weights(convert_const_pointer(resource1),
1111  resource1->id, NULL, NULL, 1,
1113  dump_node_scores(LOG_TRACE, NULL, resource1->id, r1_nodes);
1114 
1115  r2_nodes = rsc_merge_weights(convert_const_pointer(resource2),
1116  resource2->id, NULL, NULL, 1,
1118  dump_node_scores(LOG_TRACE, NULL, resource2->id, r2_nodes);
1119 
1120  /* Current location score */
1121  reason = "current location";
1122  r1_weight = -INFINITY;
1123  r2_weight = -INFINITY;
1124 
1125  if (resource1->running_on) {
1126  r1_node = pe__current_node(resource1);
1127  r1_node = g_hash_table_lookup(r1_nodes, r1_node->details->id);
1128  if (r1_node != NULL) {
1129  r1_weight = r1_node->weight;
1130  }
1131  }
1132  if (resource2->running_on) {
1133  r2_node = pe__current_node(resource2);
1134  r2_node = g_hash_table_lookup(r2_nodes, r2_node->details->id);
1135  if (r2_node != NULL) {
1136  r2_weight = r2_node->weight;
1137  }
1138  }
1139 
1140  if (r1_weight > r2_weight) {
1141  rc = -1;
1142  goto done;
1143  }
1144 
1145  if (r1_weight < r2_weight) {
1146  rc = 1;
1147  goto done;
1148  }
1149 
1150  reason = "score";
1151  for (gIter = nodes; gIter != NULL; gIter = gIter->next) {
1152  node_t *node = (node_t *) gIter->data;
1153 
1154  r1_node = NULL;
1155  r2_node = NULL;
1156 
1157  r1_weight = -INFINITY;
1158  if (r1_nodes) {
1159  r1_node = g_hash_table_lookup(r1_nodes, node->details->id);
1160  }
1161  if (r1_node) {
1162  r1_weight = r1_node->weight;
1163  }
1164 
1165  r2_weight = -INFINITY;
1166  if (r2_nodes) {
1167  r2_node = g_hash_table_lookup(r2_nodes, node->details->id);
1168  }
1169  if (r2_node) {
1170  r2_weight = r2_node->weight;
1171  }
1172 
1173  if (r1_weight > r2_weight) {
1174  rc = -1;
1175  goto done;
1176  }
1177 
1178  if (r1_weight < r2_weight) {
1179  rc = 1;
1180  goto done;
1181  }
1182  }
1183 
1184  done:
1185  crm_trace("%s (%d) on %s %c %s (%d) on %s: %s",
1186  resource1->id, r1_weight, r1_node ? r1_node->details->id : "n/a",
1187  rc < 0 ? '>' : rc > 0 ? '<' : '=',
1188  resource2->id, r2_weight, r2_node ? r2_node->details->id : "n/a", reason);
1189 
1190  if (r1_nodes) {
1191  g_hash_table_destroy(r1_nodes);
1192  }
1193  if (r2_nodes) {
1194  g_hash_table_destroy(r2_nodes);
1195  }
1196 
1197  return rc;
1198 }
1199 
1200 static void
1201 allocate_resources(pe_working_set_t * data_set)
1202 {
1203  GListPtr gIter = NULL;
1204 
1205  if (is_set(data_set->flags, pe_flag_have_remote_nodes)) {
1206  /* Force remote connection resources to be allocated first. This
1207  * also forces any colocation dependencies to be allocated as well */
1208  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1209  resource_t *rsc = (resource_t *) gIter->data;
1210  if (rsc->is_remote_node == FALSE) {
1211  continue;
1212  }
1213  pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
1214  /* For remote node connection resources, always prefer the partial
1215  * migration target during resource allocation, if the rsc is in the
1216  * middle of a migration.
1217  */
1218  rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set);
1219  }
1220  }
1221 
1222  /* now do the rest of the resources */
1223  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1224  resource_t *rsc = (resource_t *) gIter->data;
1225  if (rsc->is_remote_node == TRUE) {
1226  continue;
1227  }
1228  pe_rsc_trace(rsc, "Allocating: %s", rsc->id);
1229  rsc->cmds->allocate(rsc, NULL, data_set);
1230  }
1231 }
1232 
1233 /* We always use pe_order_preserve with these convenience functions to exempt
1234  * internally generated constraints from the prohibition of user constraints
1235  * involving remote connection resources.
1236  *
1237  * The start ordering additionally uses pe_order_runnable_left so that the
1238  * specified action is not runnable if the start is not runnable.
1239  */
1240 
1241 static inline void
1242 order_start_then_action(resource_t *lh_rsc, action_t *rh_action,
1243  enum pe_ordering extra, pe_working_set_t *data_set)
1244 {
1245  if (lh_rsc && rh_action && data_set) {
1246  custom_action_order(lh_rsc, start_key(lh_rsc), NULL,
1247  rh_action->rsc, NULL, rh_action,
1248  pe_order_preserve | pe_order_runnable_left | extra,
1249  data_set);
1250  }
1251 }
1252 
1253 static inline void
1254 order_action_then_stop(action_t *lh_action, resource_t *rh_rsc,
1255  enum pe_ordering extra, pe_working_set_t *data_set)
1256 {
1257  if (lh_action && rh_rsc && data_set) {
1258  custom_action_order(lh_action->rsc, NULL, lh_action,
1259  rh_rsc, stop_key(rh_rsc), NULL,
1260  pe_order_preserve | extra, data_set);
1261  }
1262 }
1263 
1264 static void
1265 cleanup_orphans(resource_t * rsc, pe_working_set_t * data_set)
1266 {
1267  GListPtr gIter = NULL;
1268 
1269  if (is_set(data_set->flags, pe_flag_stop_rsc_orphans) == FALSE) {
1270  return;
1271  }
1272 
1273  /* Don't recurse into ->children, those are just unallocated clone instances */
1274  if(is_not_set(rsc->flags, pe_rsc_orphan)) {
1275  return;
1276  }
1277 
1278  for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1279  node_t *node = (node_t *) gIter->data;
1280 
1281  if (node->details->online
1282  && pe_get_failcount(node, rsc, NULL, pe_fc_effective, NULL,
1283  data_set)) {
1284 
1285  pe_action_t *clear_op = NULL;
1286 
1287  clear_op = pe__clear_failcount(rsc, node, "it is orphaned",
1288  data_set);
1289 
1290  /* We can't use order_action_then_stop() here because its
1291  * pe_order_preserve breaks things
1292  */
1293  custom_action_order(clear_op->rsc, NULL, clear_op,
1294  rsc, stop_key(rsc), NULL,
1295  pe_order_optional, data_set);
1296  }
1297  }
1298 }
1299 
1300 gboolean
1302 {
1303  GListPtr gIter = NULL;
1304 
1305  if (safe_str_neq(data_set->placement_strategy, "default")) {
1306  GListPtr nodes = g_list_copy(data_set->nodes);
1307 
1308  nodes = sort_nodes_by_weight(nodes, NULL, data_set);
1309  data_set->resources =
1310  g_list_sort_with_data(data_set->resources, sort_rsc_process_order, nodes);
1311 
1312  g_list_free(nodes);
1313  }
1314 
1315  gIter = data_set->nodes;
1316  for (; gIter != NULL; gIter = gIter->next) {
1317  node_t *node = (node_t *) gIter->data;
1318 
1320  }
1321 
1322  crm_trace("Allocating services");
1323  /* Take (next) highest resource, assign it and create its actions */
1324 
1325  allocate_resources(data_set);
1326 
1327  gIter = data_set->nodes;
1328  for (; gIter != NULL; gIter = gIter->next) {
1329  node_t *node = (node_t *) gIter->data;
1330 
1331  dump_node_capacity(show_utilization ? 0 : utilization_log_level, "Remaining", node);
1332  }
1333 
1334  // Process deferred action checks
1335  pe__foreach_param_check(data_set, check_params);
1336  pe__free_param_checks(data_set);
1337 
1338  if (is_set(data_set->flags, pe_flag_startup_probes)) {
1339  crm_trace("Calculating needed probes");
1340  /* This code probably needs optimization
1341  * ptest -x with 100 nodes, 100 clones and clone-max=100:
1342 
1343  With probes:
1344 
1345  ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1346  ptest[14781]: 2010/09/27_17:56:46 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1347  ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1348  ptest[14781]: 2010/09/27_17:56:47 notice: TRACE: do_calculations: pengine.c:292 Check actions
1349  ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1350  ptest[14781]: 2010/09/27_17:56:48 notice: TRACE: stage5: allocate.c:881 Allocating services
1351  ptest[14781]: 2010/09/27_17:56:49 notice: TRACE: stage5: allocate.c:894 Calculating needed probes
1352  ptest[14781]: 2010/09/27_17:56:51 notice: TRACE: stage5: allocate.c:899 Creating actions
1353  ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: stage5: allocate.c:905 Creating done
1354  ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1355  ptest[14781]: 2010/09/27_17:56:52 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1356  36s
1357  ptest[14781]: 2010/09/27_17:57:28 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1358 
1359  Without probes:
1360 
1361  ptest[14637]: 2010/09/27_17:56:21 notice: TRACE: do_calculations: pengine.c:258 Calculate cluster status
1362  ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:278 Applying placement constraints
1363  ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:285 Create internal constraints
1364  ptest[14637]: 2010/09/27_17:56:22 notice: TRACE: do_calculations: pengine.c:292 Check actions
1365  ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: do_calculations: pengine.c:299 Allocate resources
1366  ptest[14637]: 2010/09/27_17:56:23 notice: TRACE: stage5: allocate.c:881 Allocating services
1367  ptest[14637]: 2010/09/27_17:56:24 notice: TRACE: stage5: allocate.c:899 Creating actions
1368  ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: stage5: allocate.c:905 Creating done
1369  ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:306 Processing fencing and shutdown cases
1370  ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:313 Applying ordering constraints
1371  ptest[14637]: 2010/09/27_17:56:25 notice: TRACE: do_calculations: pengine.c:320 Create transition graph
1372  */
1373 
1374  probe_resources(data_set);
1375  }
1376 
1377  crm_trace("Handle orphans");
1378 
1379  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1380  resource_t *rsc = (resource_t *) gIter->data;
1381  cleanup_orphans(rsc, data_set);
1382  }
1383 
1384  crm_trace("Creating actions");
1385 
1386  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
1387  resource_t *rsc = (resource_t *) gIter->data;
1388 
1389  rsc->cmds->create_actions(rsc, data_set);
1390  }
1391 
1392  crm_trace("Creating done");
1393  return TRUE;
1394 }
1395 
1396 static gboolean
1397 is_managed(const resource_t * rsc)
1398 {
1399  GListPtr gIter = rsc->children;
1400 
1401  if (is_set(rsc->flags, pe_rsc_managed)) {
1402  return TRUE;
1403  }
1404 
1405  for (; gIter != NULL; gIter = gIter->next) {
1406  resource_t *child_rsc = (resource_t *) gIter->data;
1407 
1408  if (is_managed(child_rsc)) {
1409  return TRUE;
1410  }
1411  }
1412 
1413  return FALSE;
1414 }
1415 
1416 static gboolean
1417 any_managed_resources(pe_working_set_t * data_set)
1418 {
1419 
1420  GListPtr gIter = data_set->resources;
1421 
1422  for (; gIter != NULL; gIter = gIter->next) {
1423  resource_t *rsc = (resource_t *) gIter->data;
1424 
1425  if (is_managed(rsc)) {
1426  return TRUE;
1427  }
1428  }
1429  return FALSE;
1430 }
1431 
1439 static void
1440 fence_guest(pe_node_t *node, pe_working_set_t *data_set)
1441 {
1442  resource_t *container = node->details->remote_rsc->container;
1443  pe_action_t *stop = NULL;
1444  pe_action_t *stonith_op = NULL;
1445 
1446  /* The fence action is just a label; we don't do anything differently for
1447  * off vs. reboot. We specify it explicitly, rather than let it default to
1448  * cluster's default action, because we are not _initiating_ fencing -- we
1449  * are creating a pseudo-event to describe fencing that is already occurring
1450  * by other means (container recovery).
1451  */
1452  const char *fence_action = "off";
1453 
1454  /* Check whether guest's container resource has any explicit stop or
1455  * start (the stop may be implied by fencing of the guest's host).
1456  */
1457  if (container) {
1458  stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
1459 
1460  if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
1461  fence_action = "reboot";
1462  }
1463  }
1464 
1465  /* Create a fence pseudo-event, so we have an event to order actions
1466  * against, and the controller can always detect it.
1467  */
1468  stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set);
1470  __FUNCTION__, __LINE__);
1471 
1472  /* We want to imply stops/demotes after the guest is stopped, not wait until
1473  * it is restarted, so we always order pseudo-fencing after stop, not start
1474  * (even though start might be closer to what is done for a real reboot).
1475  */
1476  if(stop && is_set(stop->flags, pe_action_pseudo)) {
1477  pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set);
1478  crm_info("Implying guest node %s is down (action %d) after %s fencing",
1479  node->details->uname, stonith_op->id, stop->node->details->uname);
1480  order_actions(parent_stonith_op, stonith_op,
1482 
1483  } else if (stop) {
1484  order_actions(stop, stonith_op,
1486  crm_info("Implying guest node %s is down (action %d) "
1487  "after container %s is stopped (action %d)",
1488  node->details->uname, stonith_op->id,
1489  container->id, stop->id);
1490  } else {
1491  /* If we're fencing the guest node but there's no stop for the guest
1492  * resource, we must think the guest is already stopped. However, we may
1493  * think so because its resource history was just cleaned. To avoid
1494  * unnecessarily considering the guest node down if it's really up,
1495  * order the pseudo-fencing after any stop of the connection resource,
1496  * which will be ordered after any container (re-)probe.
1497  */
1498  stop = find_first_action(node->details->remote_rsc->actions, NULL,
1499  RSC_STOP, NULL);
1500 
1501  if (stop) {
1502  order_actions(stop, stonith_op, pe_order_optional);
1503  crm_info("Implying guest node %s is down (action %d) "
1504  "after connection is stopped (action %d)",
1505  node->details->uname, stonith_op->id, stop->id);
1506  } else {
1507  /* Not sure why we're fencing, but everything must already be
1508  * cleanly stopped.
1509  */
1510  crm_info("Implying guest node %s is down (action %d) ",
1511  node->details->uname, stonith_op->id);
1512  }
1513  }
1514 
1515  /* Order/imply other actions relative to pseudo-fence as with real fence */
1516  stonith_constraints(node, stonith_op, data_set);
1517 }
1518 
1519 /*
1520  * Create dependencies for stonith and shutdown operations
1521  */
1522 gboolean
1524 {
1525  action_t *dc_down = NULL;
1526  action_t *stonith_op = NULL;
1527  gboolean integrity_lost = FALSE;
1528  gboolean need_stonith = TRUE;
1529  GListPtr gIter;
1530  GListPtr stonith_ops = NULL;
1531  GList *shutdown_ops = NULL;
1532 
1533  /* Remote ordering constraints need to happen prior to calculate
1534  * fencing because it is one more place we will mark the node as
1535  * dirty.
1536  *
1537  * A nice side-effect of doing it first is that we can remove a
1538  * bunch of special logic from apply_*_ordering() because its
1539  * already part of pe_fence_node()
1540  */
1541  crm_trace("Creating remote ordering constraints");
1542  apply_remote_node_ordering(data_set);
1543 
1544  crm_trace("Processing fencing and shutdown cases");
1545  if (any_managed_resources(data_set) == FALSE) {
1546  crm_notice("Delaying fencing operations until there are resources to manage");
1547  need_stonith = FALSE;
1548  }
1549 
1550  /* Check each node for stonith/shutdown */
1551  for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
1552  node_t *node = (node_t *) gIter->data;
1553 
1554  /* Guest nodes are "fenced" by recovering their container resource,
1555  * so handle them separately.
1556  */
1557  if (pe__is_guest_node(node)) {
1558  if (node->details->remote_requires_reset && need_stonith) {
1559  fence_guest(node, data_set);
1560  }
1561  continue;
1562  }
1563 
1564  stonith_op = NULL;
1565 
1566  if (node->details->unclean
1567  && need_stonith && pe_can_fence(data_set, node)) {
1568 
1569  stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set);
1570  pe_warn("Scheduling Node %s for STONITH", node->details->uname);
1571 
1572  stonith_constraints(node, stonith_op, data_set);
1573 
1574  if (node->details->is_dc) {
1575  // Remember if the DC is being fenced
1576  dc_down = stonith_op;
1577 
1578  } else {
1579 
1580  if (is_not_set(data_set->flags, pe_flag_concurrent_fencing)
1581  && (stonith_ops != NULL)) {
1582  /* Concurrent fencing is disabled, so order each non-DC
1583  * fencing in a chain. If there is any DC fencing or
1584  * shutdown, it will be ordered after the last action in the
1585  * chain later.
1586  */
1587  order_actions((pe_action_t *) stonith_ops->data,
1588  stonith_op, pe_order_optional);
1589  }
1590 
1591  // Remember all non-DC fencing actions in a separate list
1592  stonith_ops = g_list_prepend(stonith_ops, stonith_op);
1593  }
1594 
1595  } else if (node->details->online && node->details->shutdown &&
1596  /* TODO define what a shutdown op means for a remote node.
1597  * For now we do not send shutdown operations for remote nodes, but
1598  * if we can come up with a good use for this in the future, we will. */
1599  pe__is_guest_or_remote_node(node) == FALSE) {
1600 
1601  action_t *down_op = sched_shutdown_op(node, data_set);
1602 
1603  if (node->details->is_dc) {
1604  // Remember if the DC is being shut down
1605  dc_down = down_op;
1606  } else {
1607  // Remember non-DC shutdowns for later ordering
1608  shutdown_ops = g_list_prepend(shutdown_ops, down_op);
1609  }
1610  }
1611 
1612  if (node->details->unclean && stonith_op == NULL) {
1613  integrity_lost = TRUE;
1614  pe_warn("Node %s is unclean!", node->details->uname);
1615  }
1616  }
1617 
1618  if (integrity_lost) {
1619  if (is_set(data_set->flags, pe_flag_stonith_enabled) == FALSE) {
1620  pe_warn("YOUR RESOURCES ARE NOW LIKELY COMPROMISED");
1621  pe_err("ENABLE STONITH TO KEEP YOUR RESOURCES SAFE");
1622 
1623  } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE) {
1624  crm_notice("Cannot fence unclean nodes until quorum is"
1625  " attained (or no-quorum-policy is set to ignore)");
1626  }
1627  }
1628 
1629  if (dc_down != NULL) {
1630  /* Order any non-DC shutdowns before any DC shutdown, to avoid repeated
1631  * DC elections. However, we don't want to order non-DC shutdowns before
1632  * a DC *fencing*, because even though we don't want a node that's
1633  * shutting down to become DC, the DC fencing could be ordered before a
1634  * clone stop that's also ordered before the shutdowns, thus leading to
1635  * a graph loop.
1636  */
1637  if (safe_str_eq(dc_down->task, CRM_OP_SHUTDOWN)) {
1638  for (gIter = shutdown_ops; gIter != NULL; gIter = gIter->next) {
1639  action_t *node_stop = (action_t *) gIter->data;
1640 
1641  crm_debug("Ordering shutdown on %s before %s on DC %s",
1642  node_stop->node->details->uname,
1643  dc_down->task, dc_down->node->details->uname);
1644 
1645  order_actions(node_stop, dc_down, pe_order_optional);
1646  }
1647  }
1648 
1649  // Order any non-DC fencing before any DC fencing or shutdown
1650 
1651  if (is_set(data_set->flags, pe_flag_concurrent_fencing)) {
1652  /* With concurrent fencing, order each non-DC fencing action
1653  * separately before any DC fencing or shutdown.
1654  */
1655  for (gIter = stonith_ops; gIter != NULL; gIter = gIter->next) {
1656  order_actions((pe_action_t *) gIter->data, dc_down,
1658  }
1659  } else if (stonith_ops) {
1660  /* Without concurrent fencing, the non-DC fencing actions are
1661  * already ordered relative to each other, so we just need to order
1662  * the DC fencing after the last action in the chain (which is the
1663  * first item in the list).
1664  */
1665  order_actions((pe_action_t *) stonith_ops->data, dc_down,
1667  }
1668  }
1669  g_list_free(stonith_ops);
1670  g_list_free(shutdown_ops);
1671  return TRUE;
1672 }
1673 
1674 /*
1675  * Determine the sets of independent actions and the correct order for the
1676  * actions in each set.
1677  *
1678  * Mark dependencies of un-runnable actions un-runnable
1679  *
1680  */
1681 static GListPtr
1682 find_actions_by_task(GListPtr actions, resource_t * rsc, const char *original_key)
1683 {
1684  GListPtr list = NULL;
1685 
1686  list = find_actions(actions, original_key, NULL);
1687  if (list == NULL) {
1688  /* we're potentially searching a child of the original resource */
1689  char *key = NULL;
1690  char *task = NULL;
1691  guint interval_ms = 0;
1692 
1693  if (parse_op_key(original_key, NULL, &task, &interval_ms)) {
1694  key = generate_op_key(rsc->id, task, interval_ms);
1695  list = find_actions(actions, key, NULL);
1696 
1697  } else {
1698  crm_err("search key: %s", original_key);
1699  }
1700 
1701  free(key);
1702  free(task);
1703  }
1704 
1705  return list;
1706 }
1707 
1708 static void
1709 rsc_order_then(pe_action_t *lh_action, pe_resource_t *rsc,
1710  pe__ordering_t *order)
1711 {
1712  GListPtr gIter = NULL;
1713  GListPtr rh_actions = NULL;
1714  action_t *rh_action = NULL;
1715  enum pe_ordering type;
1716 
1717  CRM_CHECK(rsc != NULL, return);
1718  CRM_CHECK(order != NULL, return);
1719 
1720  type = order->type;
1721  rh_action = order->rh_action;
1722  crm_trace("Processing RH of ordering constraint %d", order->id);
1723 
1724  if (rh_action != NULL) {
1725  rh_actions = g_list_prepend(NULL, rh_action);
1726 
1727  } else if (rsc != NULL) {
1728  rh_actions = find_actions_by_task(rsc->actions, rsc, order->rh_action_task);
1729  }
1730 
1731  if (rh_actions == NULL) {
1732  pe_rsc_trace(rsc, "No RH-Side (%s/%s) found for constraint..."
1733  " ignoring", rsc->id, order->rh_action_task);
1734  if (lh_action) {
1735  pe_rsc_trace(rsc, "LH-Side was: %s", lh_action->uuid);
1736  }
1737  return;
1738  }
1739 
1740  if (lh_action && lh_action->rsc == rsc && is_set(lh_action->flags, pe_action_dangle)) {
1741  pe_rsc_trace(rsc, "Detected dangling operation %s -> %s", lh_action->uuid,
1742  order->rh_action_task);
1744  }
1745 
1746  gIter = rh_actions;
1747  for (; gIter != NULL; gIter = gIter->next) {
1748  action_t *rh_action_iter = (action_t *) gIter->data;
1749 
1750  if (lh_action) {
1751  order_actions(lh_action, rh_action_iter, type);
1752 
1753  } else if (type & pe_order_implies_then) {
1754  update_action_flags(rh_action_iter, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__);
1755  crm_warn("Unrunnable %s 0x%.6x", rh_action_iter->uuid, type);
1756  } else {
1757  crm_warn("neither %s 0x%.6x", rh_action_iter->uuid, type);
1758  }
1759  }
1760 
1761  g_list_free(rh_actions);
1762 }
1763 
1764 static void
1765 rsc_order_first(pe_resource_t *lh_rsc, pe__ordering_t *order,
1766  pe_working_set_t *data_set)
1767 {
1768  GListPtr gIter = NULL;
1769  GListPtr lh_actions = NULL;
1770  action_t *lh_action = order->lh_action;
1771  resource_t *rh_rsc = order->rh_rsc;
1772 
1773  crm_trace("Processing LH of ordering constraint %d", order->id);
1774  CRM_ASSERT(lh_rsc != NULL);
1775 
1776  if (lh_action != NULL) {
1777  lh_actions = g_list_prepend(NULL, lh_action);
1778 
1779  } else {
1780  lh_actions = find_actions_by_task(lh_rsc->actions, lh_rsc, order->lh_action_task);
1781  }
1782 
1783  if (lh_actions == NULL && lh_rsc != rh_rsc) {
1784  char *key = NULL;
1785  char *op_type = NULL;
1786  guint interval_ms = 0;
1787 
1788  parse_op_key(order->lh_action_task, NULL, &op_type, &interval_ms);
1789  key = generate_op_key(lh_rsc->id, op_type, interval_ms);
1790 
1791  if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_STOPPED && safe_str_eq(op_type, RSC_STOP)) {
1792  free(key);
1793  pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1794  lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1795 
1796  } else if (lh_rsc->fns->state(lh_rsc, TRUE) == RSC_ROLE_SLAVE && safe_str_eq(op_type, RSC_DEMOTE)) {
1797  free(key);
1798  pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - ignoring",
1799  lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1800 
1801  } else {
1802  pe_rsc_trace(lh_rsc, "No LH-Side (%s/%s) found for constraint %d with %s - creating",
1803  lh_rsc->id, order->lh_action_task, order->id, order->rh_action_task);
1804  lh_action = custom_action(lh_rsc, key, op_type, NULL, TRUE, TRUE, data_set);
1805  lh_actions = g_list_prepend(NULL, lh_action);
1806  }
1807 
1808  free(op_type);
1809  }
1810 
1811  gIter = lh_actions;
1812  for (; gIter != NULL; gIter = gIter->next) {
1813  action_t *lh_action_iter = (action_t *) gIter->data;
1814 
1815  if (rh_rsc == NULL && order->rh_action) {
1816  rh_rsc = order->rh_action->rsc;
1817  }
1818  if (rh_rsc) {
1819  rsc_order_then(lh_action_iter, rh_rsc, order);
1820 
1821  } else if (order->rh_action) {
1822  order_actions(lh_action_iter, order->rh_action, order->type);
1823  }
1824  }
1825 
1826  g_list_free(lh_actions);
1827 }
1828 
1829 extern void update_colo_start_chain(pe_action_t *action,
1830  pe_working_set_t *data_set);
1831 
1832 static int
1833 is_recurring_action(action_t *action)
1834 {
1835  const char *interval_ms_s = g_hash_table_lookup(action->meta,
1837  guint interval_ms = crm_parse_ms(interval_ms_s);
1838 
1839  return (interval_ms > 0);
1840 }
1841 
1842 static void
1843 apply_container_ordering(action_t *action, pe_working_set_t *data_set)
1844 {
1845  /* VMs are also classified as containers for these purposes... in
1846  * that they both involve a 'thing' running on a real or remote
1847  * cluster node.
1848  *
1849  * This allows us to be smarter about the type and extent of
1850  * recovery actions required in various scenarios
1851  */
1852  resource_t *remote_rsc = NULL;
1853  resource_t *container = NULL;
1854  enum action_tasks task = text2task(action->task);
1855 
1856  CRM_ASSERT(action->rsc);
1857  CRM_ASSERT(action->node);
1859 
1860  remote_rsc = action->node->details->remote_rsc;
1861  CRM_ASSERT(remote_rsc);
1862 
1863  container = remote_rsc->container;
1864  CRM_ASSERT(container);
1865 
1866  if(is_set(container->flags, pe_rsc_failed)) {
1867  pe_fence_node(data_set, action->node, "container failed");
1868  }
1869 
1870  crm_trace("Order %s action %s relative to %s%s for %s%s",
1871  action->task, action->uuid,
1872  is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
1873  remote_rsc->id,
1874  is_set(container->flags, pe_rsc_failed)? "failed " : "",
1875  container->id);
1876 
1877  if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
1878  || safe_str_eq(action->task, CRMD_ACTION_MIGRATED)) {
1879  /* Migration ops map to "no_action", but we need to apply the same
1880  * ordering as for stop or demote (see get_router_node()).
1881  */
1882  task = stop_rsc;
1883  }
1884 
1885  switch (task) {
1886  case start_rsc:
1887  case action_promote:
1888  /* Force resource recovery if the container is recovered */
1889  order_start_then_action(container, action, pe_order_implies_then,
1890  data_set);
1891 
1892  /* Wait for the connection resource to be up too */
1893  order_start_then_action(remote_rsc, action, pe_order_none,
1894  data_set);
1895  break;
1896 
1897  case stop_rsc:
1898  case action_demote:
1899  if (is_set(container->flags, pe_rsc_failed)) {
1900  /* When the container representing a guest node fails, any stop
1901  * or demote actions for resources running on the guest node
1902  * are implied by the container stopping. This is similar to
1903  * how fencing operations work for cluster nodes and remote
1904  * nodes.
1905  */
1906  } else {
1907  /* Ensure the operation happens before the connection is brought
1908  * down.
1909  *
1910  * If we really wanted to, we could order these after the
1911  * connection start, IFF the container's current role was
1912  * stopped (otherwise we re-introduce an ordering loop when the
1913  * connection is restarting).
1914  */
1915  order_action_then_stop(action, remote_rsc, pe_order_none,
1916  data_set);
1917  }
1918  break;
1919 
1920  default:
1921  /* Wait for the connection resource to be up */
1922  if (is_recurring_action(action)) {
1923  /* In case we ever get the recovery logic wrong, force
1924  * recurring monitors to be restarted, even if just
1925  * the connection was re-established
1926  */
1927  if(task != no_action) {
1928  order_start_then_action(remote_rsc, action,
1929  pe_order_implies_then, data_set);
1930  }
1931  } else {
1932  order_start_then_action(remote_rsc, action, pe_order_none,
1933  data_set);
1934  }
1935  break;
1936  }
1937 }
1938 
1939 static enum remote_connection_state
1940 get_remote_node_state(pe_node_t *node)
1941 {
1942  resource_t *remote_rsc = NULL;
1943  node_t *cluster_node = NULL;
1944 
1945  CRM_ASSERT(node);
1946 
1947  remote_rsc = node->details->remote_rsc;
1948  CRM_ASSERT(remote_rsc);
1949 
1950  cluster_node = pe__current_node(remote_rsc);
1951 
1952  /* If the cluster node the remote connection resource resides on
1953  * is unclean or went offline, we can't process any operations
1954  * on that remote node until after it starts elsewhere.
1955  */
1956  if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
1957  /* The connection resource is not going to run anywhere */
1958 
1959  if (cluster_node && cluster_node->details->unclean) {
1960  /* The remote connection is failed because its resource is on a
1961  * failed node and can't be recovered elsewhere, so we must fence.
1962  */
1963  return remote_state_failed;
1964  }
1965 
1966  if (is_not_set(remote_rsc->flags, pe_rsc_failed)) {
1967  /* Connection resource is cleanly stopped */
1968  return remote_state_stopped;
1969  }
1970 
1971  /* Connection resource is failed */
1972 
1973  if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
1974  && remote_rsc->remote_reconnect_ms
1975  && node->details->remote_was_fenced) {
1976 
1977  /* We won't know whether the connection is recoverable until the
1978  * reconnect interval expires and we reattempt connection.
1979  */
1980  return remote_state_unknown;
1981  }
1982 
1983  /* The remote connection is in a failed state. If there are any
1984  * resources known to be active on it (stop) or in an unknown state
1985  * (probe), we must assume the worst and fence it.
1986  */
1987  return remote_state_failed;
1988 
1989  } else if (cluster_node == NULL) {
1990  /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
1991  return remote_state_unknown;
1992 
1993  } else if(cluster_node->details->unclean == TRUE
1994  || cluster_node->details->online == FALSE) {
1995  /* Connection is running on a dead node, see if we can recover it first */
1996  return remote_state_resting;
1997 
1998  } else if (g_list_length(remote_rsc->running_on) > 1
1999  && remote_rsc->partial_migration_source
2000  && remote_rsc->partial_migration_target) {
2001  /* We're in the middle of migrating a connection resource,
2002  * wait until after the resource migrates before performing
2003  * any actions.
2004  */
2005  return remote_state_resting;
2006 
2007  }
2008  return remote_state_alive;
2009 }
2010 
2015 static void
2016 apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
2017 {
2018  resource_t *remote_rsc = NULL;
2019  enum action_tasks task = text2task(action->task);
2020  enum remote_connection_state state = get_remote_node_state(action->node);
2021 
2022  enum pe_ordering order_opts = pe_order_none;
2023 
2024  if (action->rsc == NULL) {
2025  return;
2026  }
2027 
2028  CRM_ASSERT(action->node);
2030 
2031  remote_rsc = action->node->details->remote_rsc;
2032  CRM_ASSERT(remote_rsc);
2033 
2034  crm_trace("Order %s action %s relative to %s%s (state: %s)",
2035  action->task, action->uuid,
2036  is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "",
2037  remote_rsc->id, state2text(state));
2038 
2039  if (safe_str_eq(action->task, CRMD_ACTION_MIGRATE)
2040  || safe_str_eq(action->task, CRMD_ACTION_MIGRATED)) {
2041  /* Migration ops map to "no_action", but we need to apply the same
2042  * ordering as for stop or demote (see get_router_node()).
2043  */
2044  task = stop_rsc;
2045  }
2046 
2047  switch (task) {
2048  case start_rsc:
2049  case action_promote:
2050  order_opts = pe_order_none;
2051 
2052  if (state == remote_state_failed) {
2053  /* Force recovery, by making this action required */
2054  order_opts |= pe_order_implies_then;
2055  }
2056 
2057  /* Ensure connection is up before running this action */
2058  order_start_then_action(remote_rsc, action, order_opts, data_set);
2059  break;
2060 
2061  case stop_rsc:
2062  if(state == remote_state_alive) {
2063  order_action_then_stop(action, remote_rsc,
2064  pe_order_implies_first, data_set);
2065 
2066  } else if(state == remote_state_failed) {
2067  /* We would only be here if the resource is
2068  * running on the remote node. Since we have no
2069  * way to stop it, it is necessary to fence the
2070  * node.
2071  */
2072  pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");
2073  order_action_then_stop(action, remote_rsc,
2074  pe_order_implies_first, data_set);
2075 
2076  } else if(remote_rsc->next_role == RSC_ROLE_STOPPED) {
2077  /* State must be remote_state_unknown or remote_state_stopped.
2078  * Since the connection is not coming back up in this
2079  * transition, stop this resource first.
2080  */
2081  order_action_then_stop(action, remote_rsc,
2082  pe_order_implies_first, data_set);
2083 
2084  } else {
2085  /* The connection is going to be started somewhere else, so
2086  * stop this resource after that completes.
2087  */
2088  order_start_then_action(remote_rsc, action, pe_order_none, data_set);
2089  }
2090  break;
2091 
2092  case action_demote:
2093  /* Only order this demote relative to the connection start if the
2094  * connection isn't being torn down. Otherwise, the demote would be
2095  * blocked because the connection start would not be allowed.
2096  */
2097  if(state == remote_state_resting || state == remote_state_unknown) {
2098  order_start_then_action(remote_rsc, action, pe_order_none,
2099  data_set);
2100  } /* Otherwise we can rely on the stop ordering */
2101  break;
2102 
2103  default:
2104  /* Wait for the connection resource to be up */
2105  if (is_recurring_action(action)) {
2106  /* In case we ever get the recovery logic wrong, force
2107  * recurring monitors to be restarted, even if just
2108  * the connection was re-established
2109  */
2110  order_start_then_action(remote_rsc, action,
2111  pe_order_implies_then, data_set);
2112 
2113  } else {
2114  node_t *cluster_node = pe__current_node(remote_rsc);
2115 
2116  if(task == monitor_rsc && state == remote_state_failed) {
2117  /* We would only be here if we do not know the
2118  * state of the resource on the remote node.
2119  * Since we have no way to find out, it is
2120  * necessary to fence the node.
2121  */
2122  pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");
2123  }
2124 
2125  if(cluster_node && state == remote_state_stopped) {
2126  /* The connection is currently up, but is going
2127  * down permanently.
2128  *
2129  * Make sure we check services are actually
2130  * stopped _before_ we let the connection get
2131  * closed
2132  */
2133  order_action_then_stop(action, remote_rsc,
2134  pe_order_runnable_left, data_set);
2135 
2136  } else {
2137  order_start_then_action(remote_rsc, action, pe_order_none,
2138  data_set);
2139  }
2140  }
2141  break;
2142  }
2143 }
2144 
2145 static void
2146 apply_remote_node_ordering(pe_working_set_t *data_set)
2147 {
2148  if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) {
2149  return;
2150  }
2151 
2152  for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2153  action_t *action = (action_t *) gIter->data;
2154  resource_t *remote = NULL;
2155 
2156  // We are only interested in resource actions
2157  if (action->rsc == NULL) {
2158  continue;
2159  }
2160 
2161  /* Special case: If we are clearing the failcount of an actual
2162  * remote connection resource, then make sure this happens before
2163  * any start of the resource in this transition.
2164  */
2165  if (action->rsc->is_remote_node &&
2167 
2168  custom_action_order(action->rsc,
2169  NULL,
2170  action,
2171  action->rsc,
2172  generate_op_key(action->rsc->id, RSC_START, 0),
2173  NULL,
2175  data_set);
2176 
2177  continue;
2178  }
2179 
2180  // We are only interested in actions allocated to a node
2181  if (action->node == NULL) {
2182  continue;
2183  }
2184 
2185  if (!pe__is_guest_or_remote_node(action->node)) {
2186  continue;
2187  }
2188 
2189  /* We are only interested in real actions.
2190  *
2191  * @TODO This is probably wrong; pseudo-actions might be converted to
2192  * real actions and vice versa later in update_actions() at the end of
2193  * stage7().
2194  */
2195  if (is_set(action->flags, pe_action_pseudo)) {
2196  continue;
2197  }
2198 
2199  remote = action->node->details->remote_rsc;
2200  if (remote == NULL) {
2201  // Orphaned
2202  continue;
2203  }
2204 
2205  /* Another special case: if a resource is moving to a Pacemaker Remote
2206  * node, order the stop on the original node after any start of the
2207  * remote connection. This ensures that if the connection fails to
2208  * start, we leave the resource running on the original node.
2209  */
2210  if (safe_str_eq(action->task, RSC_START)) {
2211  for (GList *item = action->rsc->actions; item != NULL;
2212  item = item->next) {
2213  pe_action_t *rsc_action = item->data;
2214 
2215  if ((rsc_action->node->details != action->node->details)
2216  && safe_str_eq(rsc_action->task, RSC_STOP)) {
2217  custom_action_order(remote, start_key(remote), NULL,
2218  action->rsc, NULL, rsc_action,
2219  pe_order_optional, data_set);
2220  }
2221  }
2222  }
2223 
2224  /* The action occurs across a remote connection, so create
2225  * ordering constraints that guarantee the action occurs while the node
2226  * is active (after start, before stop ... things like that).
2227  *
2228  * This is somewhat brittle in that we need to make sure the results of
2229  * this ordering are compatible with the result of get_router_node().
2230  * It would probably be better to add XML_LRM_ATTR_ROUTER_NODE as part
2231  * of this logic rather than action2xml().
2232  */
2233  if (remote->container) {
2234  crm_trace("Container ordering for %s", action->uuid);
2235  apply_container_ordering(action, data_set);
2236 
2237  } else {
2238  crm_trace("Remote ordering for %s", action->uuid);
2239  apply_remote_ordering(action, data_set);
2240  }
2241  }
2242 }
2243 
2244 static gboolean
2245 order_first_probe_unneeded(pe_action_t * probe, pe_action_t * rh_action)
2246 {
2247  /* No need to probe the resource on the node that is being
2248  * unfenced. Otherwise it might introduce transition loop
2249  * since probe will be performed after the node is
2250  * unfenced.
2251  */
2252  if (safe_str_eq(rh_action->task, CRM_OP_FENCE)
2253  && probe->node && rh_action->node
2254  && probe->node->details == rh_action->node->details) {
2255  const char *op = g_hash_table_lookup(rh_action->meta, "stonith_action");
2256 
2257  if (safe_str_eq(op, "on")) {
2258  return TRUE;
2259  }
2260  }
2261 
2262  // Shutdown waits for probe to complete only if it's on the same node
2263  if ((safe_str_eq(rh_action->task, CRM_OP_SHUTDOWN))
2264  && probe->node && rh_action->node
2265  && probe->node->details != rh_action->node->details) {
2266  return TRUE;
2267  }
2268  return FALSE;
2269 }
2270 
2271 
2272 static void
2273 order_first_probes(pe_working_set_t * data_set)
2274 {
2275  GListPtr gIter = NULL;
2276 
2277  for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
2278  pe__ordering_t *order = gIter->data;
2279  enum pe_ordering order_type = pe_order_optional;
2280 
2281  pe_resource_t *lh_rsc = order->lh_rsc;
2282  pe_resource_t *rh_rsc = order->rh_rsc;
2283  pe_action_t *lh_action = order->lh_action;
2284  pe_action_t *rh_action = order->rh_action;
2285  const char *lh_action_task = order->lh_action_task;
2286  const char *rh_action_task = order->rh_action_task;
2287 
2288  GListPtr probes = NULL;
2289  GListPtr rh_actions = NULL;
2290 
2291  GListPtr pIter = NULL;
2292 
2293  if (lh_rsc == NULL) {
2294  continue;
2295 
2296  } else if (rh_rsc && lh_rsc == rh_rsc) {
2297  continue;
2298  }
2299 
2300  if (lh_action == NULL && lh_action_task == NULL) {
2301  continue;
2302  }
2303 
2304  if (rh_action == NULL && rh_action_task == NULL) {
2305  continue;
2306  }
2307 
2308  /* Technically probe is expected to return "not running", which could be
2309  * the alternative of stop action if the status of the resource is
2310  * unknown yet.
2311  */
2312  if (lh_action && safe_str_neq(lh_action->task, RSC_STOP)) {
2313  continue;
2314 
2315  } else if (lh_action == NULL
2316  && lh_action_task
2317  && crm_ends_with(lh_action_task, "_" RSC_STOP "_0") == FALSE) {
2318  continue;
2319  }
2320 
2321  /* Do not probe the resource inside of a stopping container. Otherwise
2322  * it might introduce transition loop since probe will be performed
2323  * after the container starts again.
2324  */
2325  if (rh_rsc && lh_rsc->container == rh_rsc) {
2326  if (rh_action && safe_str_eq(rh_action->task, RSC_STOP)) {
2327  continue;
2328 
2329  } else if (rh_action == NULL && rh_action_task
2330  && crm_ends_with(rh_action_task,"_" RSC_STOP "_0")) {
2331  continue;
2332  }
2333  }
2334 
2335  if (order->type == pe_order_none) {
2336  continue;
2337  }
2338 
2339  // Preserve the order options for future filtering
2340  if (is_set(order->type, pe_order_apply_first_non_migratable)) {
2342  }
2343 
2344  if (is_set(order->type, pe_order_same_node)) {
2345  set_bit(order_type, pe_order_same_node);
2346  }
2347 
2348  // Keep the order types for future filtering
2349  if (order->type == pe_order_anti_colocation
2350  || order->type == pe_order_load) {
2351  order_type = order->type;
2352  }
2353 
2354  probes = pe__resource_actions(lh_rsc, NULL, RSC_STATUS, FALSE);
2355  if (probes == NULL) {
2356  continue;
2357  }
2358 
2359  if (rh_action) {
2360  rh_actions = g_list_prepend(rh_actions, rh_action);
2361 
2362  } else if (rh_rsc && rh_action_task) {
2363  rh_actions = find_actions(rh_rsc->actions, rh_action_task, NULL);
2364  }
2365 
2366  if (rh_actions == NULL) {
2367  g_list_free(probes);
2368  continue;
2369  }
2370 
2371  crm_trace("Processing for LH probe based on ordering constraint %s -> %s"
2372  " (id=%d, type=%.6x)",
2373  lh_action ? lh_action->uuid : lh_action_task,
2374  rh_action ? rh_action->uuid : rh_action_task,
2375  order->id, order->type);
2376 
2377  for (pIter = probes; pIter != NULL; pIter = pIter->next) {
2378  pe_action_t *probe = (pe_action_t *) pIter->data;
2379  GListPtr rIter = NULL;
2380 
2381  for (rIter = rh_actions; rIter != NULL; rIter = rIter->next) {
2382  pe_action_t *rh_action_iter = (pe_action_t *) rIter->data;
2383 
2384  if (order_first_probe_unneeded(probe, rh_action_iter)) {
2385  continue;
2386  }
2387  order_actions(probe, rh_action_iter, order_type);
2388  }
2389  }
2390 
2391  g_list_free(rh_actions);
2392  g_list_free(probes);
2393  }
2394 }
2395 
2396 static void
2397 order_then_probes(pe_working_set_t * data_set)
2398 {
2399 #if 0
2400  GListPtr gIter = NULL;
2401 
2402  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2403  resource_t *rsc = (resource_t *) gIter->data;
2404 
2405  /* Given "A then B", we would prefer to wait for A to be
2406  * started before probing B.
2407  *
2408  * If A was a filesystem on which the binaries and data for B
2409  * lived, it would have been useful if the author of B's agent
2410  * could assume that A is running before B.monitor will be
2411  * called.
2412  *
2413  * However we can't _only_ probe once A is running, otherwise
2414  * we'd not detect the state of B if A could not be started
2415  * for some reason.
2416  *
2417  * In practice however, we cannot even do an opportunistic
2418  * version of this because B may be moving:
2419  *
2420  * B.probe -> B.start
2421  * B.probe -> B.stop
2422  * B.stop -> B.start
2423  * A.stop -> A.start
2424  * A.start -> B.probe
2425  *
2426  * So far so good, but if we add the result of this code:
2427  *
2428  * B.stop -> A.stop
2429  *
2430  * Then we get a loop:
2431  *
2432  * B.probe -> B.stop -> A.stop -> A.start -> B.probe
2433  *
2434  * We could kill the 'B.probe -> B.stop' dependency, but that
2435  * could mean stopping B "too" soon, because B.start must wait
2436  * for the probes to complete.
2437  *
2438  * Another option is to allow it only if A is a non-unique
2439  * clone with clone-max == node-max (since we'll never be
2440  * moving it). However, we could still be stopping one
2441  * instance at the same time as starting another.
2442 
2443  * The complexity of checking for allowed conditions combined
2444  * with the ever narrowing usecase suggests that this code
2445  * should remain disabled until someone gets smarter.
2446  */
2447  action_t *start = NULL;
2448  GListPtr actions = NULL;
2449  GListPtr probes = NULL;
2450 
2451  actions = pe__resource_actions(rsc, NULL, RSC_START, FALSE);
2452 
2453  if (actions) {
2454  start = actions->data;
2455  g_list_free(actions);
2456  }
2457 
2458  if(start == NULL) {
2459  crm_err("No start action for %s", rsc->id);
2460  continue;
2461  }
2462 
2463  probes = pe__resource_actions(rsc, NULL, RSC_STATUS, FALSE);
2464 
2465  for (actions = start->actions_before; actions != NULL; actions = actions->next) {
2466  action_wrapper_t *before = (action_wrapper_t *) actions->data;
2467 
2468  GListPtr pIter = NULL;
2469  action_t *first = before->action;
2470  resource_t *first_rsc = first->rsc;
2471 
2472  if(first->required_runnable_before) {
2473  GListPtr clone_actions = NULL;
2474  for (clone_actions = first->actions_before; clone_actions != NULL; clone_actions = clone_actions->next) {
2475  before = (action_wrapper_t *) clone_actions->data;
2476 
2477  crm_trace("Testing %s -> %s (%p) for %s", first->uuid, before->action->uuid, before->action->rsc, start->uuid);
2478 
2479  CRM_ASSERT(before->action->rsc);
2480  first_rsc = before->action->rsc;
2481  break;
2482  }
2483 
2484  } else if(safe_str_neq(first->task, RSC_START)) {
2485  crm_trace("Not a start op %s for %s", first->uuid, start->uuid);
2486  }
2487 
2488  if(first_rsc == NULL) {
2489  continue;
2490 
2491  } else if(uber_parent(first_rsc) == uber_parent(start->rsc)) {
2492  crm_trace("Same parent %s for %s", first_rsc->id, start->uuid);
2493  continue;
2494 
2495  } else if(FALSE && pe_rsc_is_clone(uber_parent(first_rsc)) == FALSE) {
2496  crm_trace("Not a clone %s for %s", first_rsc->id, start->uuid);
2497  continue;
2498  }
2499 
2500  crm_err("Applying %s before %s %d", first->uuid, start->uuid, uber_parent(first_rsc)->variant);
2501 
2502  for (pIter = probes; pIter != NULL; pIter = pIter->next) {
2503  action_t *probe = (action_t *) pIter->data;
2504 
2505  crm_err("Ordering %s before %s", first->uuid, probe->uuid);
2506  order_actions(first, probe, pe_order_optional);
2507  }
2508  }
2509  }
2510 #endif
2511 }
2512 
2513 static void
2514 order_probes(pe_working_set_t * data_set)
2515 {
2516  order_first_probes(data_set);
2517  order_then_probes(data_set);
2518 }
2519 
2520 gboolean
2522 {
2523  GListPtr gIter = NULL;
2524 
2525  crm_trace("Applying ordering constraints");
2526 
2527  /* Don't ask me why, but apparently they need to be processed in
2528  * the order they were created in... go figure
2529  *
2530  * Also g_list_append() has horrendous performance characteristics
2531  * So we need to use g_list_prepend() and then reverse the list here
2532  */
2533  data_set->ordering_constraints = g_list_reverse(data_set->ordering_constraints);
2534 
2535  for (gIter = data_set->ordering_constraints; gIter != NULL; gIter = gIter->next) {
2536  pe__ordering_t *order = gIter->data;
2537  resource_t *rsc = order->lh_rsc;
2538 
2539  crm_trace("Applying ordering constraint: %d", order->id);
2540 
2541  if (rsc != NULL) {
2542  crm_trace("rsc_action-to-*");
2543  rsc_order_first(rsc, order, data_set);
2544  continue;
2545  }
2546 
2547  rsc = order->rh_rsc;
2548  if (rsc != NULL) {
2549  crm_trace("action-to-rsc_action");
2550  rsc_order_then(order->lh_action, rsc, order);
2551 
2552  } else {
2553  crm_trace("action-to-action");
2554  order_actions(order->lh_action, order->rh_action, order->type);
2555  }
2556  }
2557 
2558  for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2559  action_t *action = (action_t *) gIter->data;
2560 
2561  update_colo_start_chain(action, data_set);
2562  }
2563 
2564  crm_trace("Ordering probes");
2565  order_probes(data_set);
2566 
2567  crm_trace("Updating %d actions", g_list_length(data_set->actions));
2568  for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2569  action_t *action = (action_t *) gIter->data;
2570 
2571  update_action(action, data_set);
2572  }
2573 
2574  LogNodeActions(data_set, FALSE);
2575  for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
2576  resource_t *rsc = (resource_t *) gIter->data;
2577 
2578  LogActions(rsc, data_set, FALSE);
2579  }
2580  return TRUE;
2581 }
2582 
2583 static int transition_id = -1;
2584 
2591 void
2592 pcmk__log_transition_summary(const char *filename)
2593 {
2594  if (was_processing_error) {
2595  crm_err("Calculated transition %d (with errors), saving inputs in %s",
2596  transition_id, filename);
2597 
2598  } else if (was_processing_warning) {
2599  crm_warn("Calculated transition %d (with warnings), saving inputs in %s",
2600  transition_id, filename);
2601 
2602  } else {
2603  crm_notice("Calculated transition %d, saving inputs in %s",
2604  transition_id, filename);
2605  }
2606  if (crm_config_error) {
2607  crm_notice("Configuration errors found during scheduler processing,"
2608  " please run \"crm_verify -L\" to identify issues");
2609  }
2610 }
2611 
2612 /*
2613  * Create a dependency graph to send to the transitioner (via the controller)
2614  */
2615 gboolean
2617 {
2618  GListPtr gIter = NULL;
2619  const char *value = NULL;
2620 
2621  transition_id++;
2622  crm_trace("Creating transition graph %d.", transition_id);
2623 
2624  data_set->graph = create_xml_node(NULL, XML_TAG_GRAPH);
2625 
2626  value = pe_pref(data_set->config_hash, "cluster-delay");
2627  crm_xml_add(data_set->graph, "cluster-delay", value);
2628 
2629  value = pe_pref(data_set->config_hash, "stonith-timeout");
2630  crm_xml_add(data_set->graph, "stonith-timeout", value);
2631 
2632  crm_xml_add(data_set->graph, "failed-stop-offset", "INFINITY");
2633 
2634  if (is_set(data_set->flags, pe_flag_start_failure_fatal)) {
2635  crm_xml_add(data_set->graph, "failed-start-offset", "INFINITY");
2636  } else {
2637  crm_xml_add(data_set->graph, "failed-start-offset", "1");
2638  }
2639 
2640  value = pe_pref(data_set->config_hash, "batch-limit");
2641  crm_xml_add(data_set->graph, "batch-limit", value);
2642 
2643  crm_xml_add_int(data_set->graph, "transition_id", transition_id);
2644 
2645  value = pe_pref(data_set->config_hash, "migration-limit");
2646  if (crm_int_helper(value, NULL) > 0) {
2647  crm_xml_add(data_set->graph, "migration-limit", value);
2648  }
2649 
2650 /* errors...
2651  slist_iter(action, action_t, action_list, lpc,
2652  if(action->optional == FALSE && action->runnable == FALSE) {
2653  print_action("Ignoring", action, TRUE);
2654  }
2655  );
2656 */
2657 
2658  gIter = data_set->resources;
2659  for (; gIter != NULL; gIter = gIter->next) {
2660  resource_t *rsc = (resource_t *) gIter->data;
2661 
2662  pe_rsc_trace(rsc, "processing actions for rsc=%s", rsc->id);
2663  rsc->cmds->expand(rsc, data_set);
2664  }
2665 
2666  crm_log_xml_trace(data_set->graph, "created resource-driven action list");
2667 
2668  /* pseudo action to distribute list of nodes with maintenance state update */
2669  add_maintenance_update(data_set);
2670 
2671  /* catch any non-resource specific actions */
2672  crm_trace("processing non-resource actions");
2673 
2674  gIter = data_set->actions;
2675  for (; gIter != NULL; gIter = gIter->next) {
2676  action_t *action = (action_t *) gIter->data;
2677 
2678  if (action->rsc
2679  && action->node
2680  && action->node->details->shutdown
2681  && is_not_set(action->rsc->flags, pe_rsc_maintenance)
2682  && is_not_set(action->flags, pe_action_optional)
2683  && is_not_set(action->flags, pe_action_runnable)
2684  && crm_str_eq(action->task, RSC_STOP, TRUE)
2685  ) {
2686  /* Eventually we should just ignore the 'fence' case
2687  * But for now it's the best way to detect (in CTS) when
2688  * CIB resource updates are being lost
2689  */
2690  if (is_set(data_set->flags, pe_flag_have_quorum)
2691  || data_set->no_quorum_policy == no_quorum_ignore) {
2692  crm_crit("Cannot %s node '%s' because of %s:%s%s (%s)",
2693  action->node->details->unclean ? "fence" : "shut down",
2694  action->node->details->uname, action->rsc->id,
2695  is_not_set(action->rsc->flags, pe_rsc_managed) ? " unmanaged" : " blocked",
2696  is_set(action->rsc->flags, pe_rsc_failed) ? " failed" : "",
2697  action->uuid);
2698  }
2699  }
2700 
2701  graph_element_from_action(action, data_set);
2702  }
2703 
2704  crm_log_xml_trace(data_set->graph, "created generic action list");
2705  crm_trace("Created transition graph %d.", transition_id);
2706 
2707  return TRUE;
2708 }
2709 
2710 void
2711 LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
2712 {
2713  GListPtr gIter = NULL;
2714 
2715  for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
2716  char *node_name = NULL;
2717  char *task = NULL;
2718  action_t *action = (action_t *) gIter->data;
2719 
2720  if (action->rsc != NULL) {
2721  continue;
2722  } else if (is_set(action->flags, pe_action_optional)) {
2723  continue;
2724  }
2725 
2726  if (pe__is_guest_node(action->node)) {
2727  node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
2728  } else if(action->node) {
2729  node_name = crm_strdup_printf("%s", action->node->details->uname);
2730  }
2731 
2732 
2733  if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
2734  task = strdup("Shutdown");
2735  } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
2736  const char *op = g_hash_table_lookup(action->meta, "stonith_action");
2737  task = crm_strdup_printf("Fence (%s)", op);
2738  }
2739 
2740  if(task == NULL) {
2741  /* Nothing to report */
2742  } else if(terminal && action->reason) {
2743  printf(" * %s %s '%s'\n", task, node_name, action->reason);
2744  } else if(terminal) {
2745  printf(" * %s %s\n", task, node_name);
2746  } else if(action->reason) {
2747  crm_notice(" * %s %s '%s'\n", task, node_name, action->reason);
2748  } else {
2749  crm_notice(" * %s %s\n", task, node_name);
2750  }
2751 
2752  free(node_name);
2753  free(task);
2754  }
2755 }
pe_action_t * pe_cancel_op(pe_resource_t *rsc, const char *name, guint interval_ms, pe_node_t *node, pe_working_set_t *data_set)
#define LOG_TRACE
Definition: logging.h:26
void pe__foreach_param_check(pe_working_set_t *data_set, void(*cb)(pe_resource_t *, pe_node_t *, xmlNode *, enum pe_check_parameters, pe_working_set_t *))
Definition: remote.c:241
#define CRM_CHECK(expr, failure_action)
Definition: logging.h:156
GListPtr nodes
Definition: pe_types.h:133
node_t * group_color(resource_t *rsc, node_t *preferred, pe_working_set_t *data_set)
xmlNode * find_xml_node(xmlNode *cib, const char *node_path, gboolean must_find)
Definition: xml.c:1678
void group_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
resource_t * lh_rsc
Definition: internal.h:40
pe_node_t * pcmk__bundle_color(pe_resource_t *rsc, pe_node_t *preferred, pe_working_set_t *data_set)
enum pe_quorum_policy no_quorum_policy
Definition: pe_types.h:125
#define RSC_STOP
Definition: crm.h:177
A dumping ground.
gboolean parse_op_key(const char *key, char **rsc_id, char **op_type, guint *interval_ms)
Definition: operations.c:47
GHashTable * pcmk__bundle_merge_weights(pe_resource_t *rsc, const char *rhs, GHashTable *nodes, const char *attr, float factor, enum pe_weights flags)
#define crm_notice(fmt, args...)
Definition: logging.h:242
#define CRMD_ACTION_MIGRATED
Definition: crm.h:147
void native_expand(resource_t *rsc, pe_working_set_t *data_set)
#define pe_flag_stop_action_orphans
Definition: pe_types.h:97
GHashTable * attrs
Definition: pe_types.h:204
enum pe_action_flags pcmk__bundle_action_flags(pe_action_t *action, pe_node_t *node)
#define pe_rsc_debug(rsc, fmt, args...)
Definition: internal.h:17
#define crm_crit(fmt, args...)
Definition: logging.h:239
gboolean safe_str_neq(const char *a, const char *b)
Definition: strings.c:157
#define INFINITY
Definition: crm.h:73
void native_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
gboolean stage2(pe_working_set_t *data_set)
gboolean stage5(pe_working_set_t *data_set)
#define CRM_OP_FENCE
Definition: crm.h:119
action_t * rh_action
Definition: internal.h:46
gboolean stage3(pe_working_set_t *data_set)
#define XML_ATTR_TRANSITION_MAGIC
Definition: msg_xml.h:356
enum rsc_role_e(* state)(const pe_resource_t *, gboolean)
Definition: pe_types.h:51
node_t * clone_color(resource_t *rsc, node_t *preferred, pe_working_set_t *data_set)
GList * sort_nodes_by_weight(GList *nodes, pe_node_t *active_node, pe_working_set_t *data_set)
pe_check_parameters
Definition: pe_types.h:165
#define XML_TAG_GRAPH
Definition: msg_xml.h:288
#define stop_action(rsc, node, optional)
Definition: internal.h:210
void pe__add_param_check(xmlNode *rsc_op, pe_resource_t *rsc, pe_node_t *node, enum pe_check_parameters, pe_working_set_t *data_set)
Definition: remote.c:215
pe_resource_t * container
Definition: pe_types.h:343
pe_node_t * partial_migration_source
Definition: pe_types.h:328
#define pe_flag_concurrent_fencing
Definition: pe_types.h:94
#define XML_ATTR_TYPE
Definition: msg_xml.h:99
void group_expand(resource_t *rsc, pe_working_set_t *data_set)
bool pe_can_fence(pe_working_set_t *data_set, node_t *node)
Definition: utils.c:91
void(* expand)(resource_t *, pe_working_set_t *)
GHashTable * clone_merge_weights(resource_t *rsc, const char *rhs, GHashTable *nodes, const char *attr, float factor, enum pe_weights flags)
void pcmk__log_transition_summary(const char *filename)
int utilization_log_level
#define XML_CIB_TAG_CONSTRAINTS
Definition: msg_xml.h:143
#define CRM_OP_REPROBE
Definition: crm.h:128
resource_alloc_functions_t * cmds
Definition: pe_types.h:296
#define pe_flag_symmetric_cluster
Definition: pe_types.h:88
xmlNode * get_object_root(const char *object_type, xmlNode *the_root)
Definition: cib_utils.c:152
void clone_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
gboolean stage0(pe_working_set_t *data_set)
xmlNode * xml
Definition: pe_types.h:286
pe_resource_t * rsc
Definition: pe_types.h:365
enum rsc_role_e next_role
Definition: pe_types.h:334
action_t * pe_fence_op(node_t *node, const char *op, bool optional, const char *reason, pe_working_set_t *data_set)
Definition: utils.c:2193
void add_maintenance_update(pe_working_set_t *data_set)
const char * crm_xml_add_int(xmlNode *node, const char *name, int value)
Create an XML attribute with specified name and integer value.
Definition: nvpair.c:383
gboolean exclusive_discover
Definition: pe_types.h:315
long long crm_int_helper(const char *text, char **end_text)
Definition: strings.c:34
int char2score(const char *score)
Definition: utils.c:199
pe_resource_t * remote_rsc
Definition: pe_types.h:200
pe_action_t * sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set)
resource_alloc_functions_t resource_class_alloc_functions[]
GListPtr find_actions(GListPtr input, const char *key, const node_t *on_node)
Definition: utils.c:1428
#define pe_rsc_unique
Definition: pe_types.h:223
resource_object_functions_t * fns
Definition: pe_types.h:295
#define XML_LRM_TAG_RESOURCE
Definition: msg_xml.h:227
void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite)
Definition: utils.c:2345
const char * crm_xml_add(xmlNode *node, const char *name, const char *value)
Create an XML attribute with specified name and value.
Definition: nvpair.c:275
remote_connection_state
gboolean pe__is_guest_or_remote_node(pe_node_t *node)
Definition: remote.c:58
int crm_parse_int(const char *text, const char *default_text)
Parse an integer value from a string.
Definition: strings.c:110
void pcmk__bundle_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
GListPtr resources
Definition: pe_types.h:134
gint sort_op_by_callid(gconstpointer a, gconstpointer b)
Definition: utils.c:1609
gboolean unpack_constraints(xmlNode *xml_constraints, pe_working_set_t *data_set)
#define pe_flag_have_status
Definition: pe_types.h:105
gboolean clone_create_probe(resource_t *rsc, node_t *node, action_t *complete, gboolean force, pe_working_set_t *data_set)
void group_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
enum action_tasks text2task(const char *task)
Definition: common.c:224
void native_create_actions(resource_t *rsc, pe_working_set_t *data_set)
void(* create_actions)(resource_t *, pe_working_set_t *)
const char * pe_pref(GHashTable *options, const char *name)
Definition: common.c:181
enum pe_action_flags clone_action_flags(action_t *action, node_t *node)
xmlNode * params_restart
Definition: internal.h:312
enum pe_graph_flags native_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node, enum pe_action_flags flags, enum pe_action_flags filter, enum pe_ordering type, pe_working_set_t *data_set)
#define clear_bit(word, bit)
Definition: crm_internal.h:168
#define XML_CIB_TAG_LRM
Definition: msg_xml.h:225
void native_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
pe_resource_t * rsc_lh
Definition: internal.h:29
pe_node_t * partial_migration_target
Definition: pe_types.h:327
#define RSC_START
Definition: crm.h:174
int migration_threshold
Definition: pe_types.h:307
pe_node_t * allocated_to
Definition: pe_types.h:326
pe_action_t * action
Definition: pe_types.h:483
gboolean remote_was_fenced
Definition: pe_types.h:195
#define pe_flag_have_quorum
Definition: pe_types.h:87
bool pe__bundle_needs_remote_name(pe_resource_t *rsc)
Definition: bundle.c:955
gboolean remote_requires_reset
Definition: pe_types.h:194
char * reason
Definition: pe_types.h:372
bool crm_starts_with(const char *str, const char *prefix)
Check whether a string starts with a certain sequence.
Definition: strings.c:259
pe_node_t * pe_find_node_id(GListPtr node_list, const char *id)
Definition: status.c:396
#define CRM_TRACE_INIT_DATA(name)
Definition: logging.h:111
#define CRMD_ACTION_START
Definition: crm.h:149
void group_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
gboolean is_dc
Definition: pe_types.h:191
void native_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
GListPtr placement_constraints
Definition: pe_types.h:135
void native_append_meta(resource_t *rsc, xmlNode *xml)
#define XML_LRM_ATTR_TASK
Definition: msg_xml.h:260
pe__location_t * rsc2node_new(const char *id, pe_resource_t *rsc, int weight, const char *discovery_mode, pe_node_t *node, pe_working_set_t *data_set)
#define CRMD_ACTION_STOP
Definition: crm.h:152
GHashTable * group_merge_weights(resource_t *rsc, const char *rhs, GHashTable *nodes, const char *attr, float factor, enum pe_weights flags)
#define CRM_OP_CLEAR_FAILCOUNT
Definition: crm.h:129
#define pe_warn(fmt...)
Definition: internal.h:21
int weight
Definition: pe_types.h:210
gboolean crm_config_error
Definition: utils.c:59
#define pe_flag_have_remote_nodes
Definition: pe_types.h:106
#define crm_warn(fmt, args...)
Definition: logging.h:241
guint remote_reconnect_ms
Definition: pe_types.h:308
#define set_bit(word, bit)
Definition: crm_internal.h:167
pe_action_flags
Definition: pe_types.h:258
#define CRM_OP_FMT
Definition: crm_internal.h:133
#define pe_rsc_failed
Definition: pe_types.h:234
gboolean pe__is_guest_node(pe_node_t *node)
Definition: remote.c:47
#define crm_debug(fmt, args...)
Definition: logging.h:245
gboolean stage8(pe_working_set_t *data_set)
pe_resource_t * uber_parent(pe_resource_t *rsc)
Definition: complex.c:746
#define pe_flag_sanitized
Definition: pe_types.h:109
#define XML_ATTR_ID
Definition: msg_xml.h:96
const char * crm_element_value(const xmlNode *data, const char *name)
Retrieve the value of an XML attribute.
Definition: nvpair.c:423
void clone_expand(resource_t *rsc, pe_working_set_t *data_set)
#define XML_BOOLEAN_TRUE
Definition: msg_xml.h:107
#define XML_CIB_TAG_STATE
Definition: msg_xml.h:158
#define stop_key(rsc)
Definition: internal.h:209
enum pe_graph_flags group_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node, enum pe_action_flags flags, enum pe_action_flags filter, enum pe_ordering type, pe_working_set_t *data_set)
#define pe_rsc_start_pending
Definition: pe_types.h:236
char * task
Definition: pe_types.h:369
gboolean stage4(pe_working_set_t *data_set)
void group_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
gboolean update_action(pe_action_t *action, pe_working_set_t *data_set)
#define crm_trace(fmt, args...)
Definition: logging.h:246
enum rsc_digest_cmp_val rc
Definition: internal.h:309
action_t * find_first_action(GListPtr input, const char *uuid, const char *task, node_t *on_node)
Definition: utils.c:1398
char * digest_secure_calc
Definition: internal.h:314
void calculate_active_ops(GList *sorted_op_list, int *start_index, int *stop_index)
Definition: unpack.c:2117
GHashTable * meta
Definition: pe_types.h:379
struct pe_node_shared_s * details
Definition: pe_types.h:213
GListPtr running_on
Definition: pe_types.h:329
pe_node_t * node
Definition: pe_types.h:366
GHashTable * rsc_merge_weights(resource_t *rsc, const char *rhs, GHashTable *nodes, const char *attr, float factor, enum pe_weights flags)
#define XML_AGENT_ATTR_PROVIDER
Definition: msg_xml.h:230
unsigned long long flags
Definition: pe_types.h:311
const char * uname
Definition: pe_types.h:179
void LogActions(resource_t *rsc, pe_working_set_t *data_set, gboolean terminal)
void pe_fence_node(pe_working_set_t *data_set, node_t *node, const char *reason)
Schedule a fence action for a node.
Definition: unpack.c:70
GListPtr actions
Definition: pe_types.h:140
Wrappers for and extensions to libxml2.
#define XML_ATTR_TE_NOWAIT
Definition: msg_xml.h:359
GHashTable * config_hash
Definition: pe_types.h:127
char * clone_name
Definition: pe_types.h:285
void LogNodeActions(pe_working_set_t *data_set, gboolean terminal)
xmlNode * create_xml_node(xmlNode *parent, const char *name)
Definition: xml.c:1890
action_t * custom_action(resource_t *rsc, char *key, const char *task, node_t *on_node, gboolean optional, gboolean foo, pe_working_set_t *data_set)
Definition: utils.c:441
#define pe_flag_stonith_enabled
Definition: pe_types.h:91
const char * pe_node_attribute_raw(pe_node_t *node, const char *name)
Definition: common.c:462
xmlNode * find_rsc_op_entry(resource_t *rsc, const char *key)
Definition: utils.c:1247
enum pe_ordering type
Definition: internal.h:37
void clone_internal_constraints(resource_t *rsc, pe_working_set_t *data_set)
char * uuid
Definition: pe_types.h:370
#define XML_LRM_ATTR_RESTART_DIGEST
Definition: msg_xml.h:276
gboolean native_create_probe(resource_t *rsc, node_t *node, action_t *complete, gboolean force, pe_working_set_t *data_set)
enum pe_obj_types variant
Definition: pe_types.h:293
gboolean xml_has_children(const xmlNode *root)
Definition: xml.c:3206
xmlNode * input
Definition: pe_types.h:113
void ReloadRsc(resource_t *rsc, node_t *node, pe_working_set_t *data_set)
gboolean crm_str_eq(const char *a, const char *b, gboolean use_case)
Definition: strings.c:220
const char * placement_strategy
Definition: pe_types.h:120
gboolean can_run_resources(const node_t *node)
void graph_element_from_action(action_t *action, pe_working_set_t *data_set)
int rsc_discover_mode
Definition: pe_types.h:214
xmlNode * params_all
Definition: internal.h:310
GListPtr actions
Definition: pe_types.h:322
#define CRM_OP_SHUTDOWN
Definition: crm.h:118
void(* rsc_location)(pe_resource_t *, pe__location_t *)
void pe__free_param_checks(pe_working_set_t *data_set)
Definition: remote.c:256
const char * id
Definition: pe_types.h:178
gboolean crm_ends_with(const char *s, const char *match)
Definition: strings.c:309
gboolean pcmk__bundle_create_probe(pe_resource_t *rsc, pe_node_t *node, pe_action_t *complete, gboolean force, pe_working_set_t *data_set)
GListPtr ordering_constraints
Definition: pe_types.h:136
enum pe_action_flags group_action_flags(action_t *action, node_t *node)
gboolean rsc_discovery_enabled
Definition: pe_types.h:193
#define XML_LRM_ATTR_SECURE_DIGEST
Definition: msg_xml.h:277
gboolean show_utilization
guint crm_parse_ms(const char *text)
Definition: strings.c:143
node_t * native_color(resource_t *rsc, node_t *preferred, pe_working_set_t *data_set)
void update_colo_start_chain(pe_action_t *action, pe_working_set_t *data_set)
gboolean stage7(pe_working_set_t *data_set)
Cluster status and scheduling.
gboolean is_remote_node
Definition: pe_types.h:314
GListPtr children
Definition: pe_types.h:340
void add_hash_param(GHashTable *hash, const char *name, const char *value)
Definition: common.c:406
void complex_set_cmds(resource_t *rsc)
void(* internal_constraints)(resource_t *, pe_working_set_t *)
void pcmk__bundle_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
gboolean update_action_flags(action_t *action, enum pe_action_flags flags, const char *source, int line)
void clone_rsc_location(pe_resource_t *rsc, pe__location_t *constraint)
void clone_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
#define XML_LRM_TAG_RESOURCES
Definition: msg_xml.h:226
#define crm_err(fmt, args...)
Definition: logging.h:240
void resource_location(resource_t *rsc, node_t *node, int score, const char *tag, pe_working_set_t *data_set)
Definition: utils.c:1565
#define CRM_ASSERT(expr)
Definition: results.h:42
resource_t * rh_rsc
Definition: internal.h:45
#define RSC_STATUS
Definition: crm.h:188
pe_action_t * pe__clear_failcount(pe_resource_t *rsc, pe_node_t *node, const char *reason, pe_working_set_t *data_set)
Schedule a controller operation to clear a fail count.
Definition: failcounts.c:360
Cluster Configuration.
#define RSC_PROMOTE
Definition: crm.h:180
gboolean cluster_status(pe_working_set_t *data_set)
Definition: status.c:68
void group_create_actions(resource_t *rsc, pe_working_set_t *data_set)
gboolean DeleteRsc(resource_t *rsc, node_t *node, gboolean optional, pe_working_set_t *data_set)
GHashTable * native_merge_weights(resource_t *rsc, const char *rhs, GHashTable *nodes, const char *attr, float factor, enum pe_weights flags)
#define XML_LRM_ATTR_INTERVAL_MS
Definition: msg_xml.h:258
gboolean stage6(pe_working_set_t *data_set)
#define crm_log_xml_info(xml, text)
Definition: logging.h:252
#define DIMOF(a)
Definition: crm.h:35
void dump_node_capacity(int level, const char *comment, node_t *node)
Definition: utils.c:353
int merge_weights(int w1, int w2)
Definition: common.c:369
#define XML_LRM_ATTR_CALLID
Definition: msg_xml.h:272
#define CRMD_ACTION_MIGRATE
Definition: crm.h:146
int node_score_red
Definition: utils.c:63
gboolean shutdown
Definition: pe_types.h:189
char data[0]
Definition: internal.h:92
#define crm_str(x)
Definition: logging.h:266
void pcmk__bundle_expand(pe_resource_t *rsc, pe_working_set_t *data_set)
int pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure, uint32_t flags, xmlNode *xml_op, pe_working_set_t *data_set)
Definition: failcounts.c:251
gboolean(* create_probe)(resource_t *, node_t *, action_t *, gboolean, pe_working_set_t *)
#define pe_flag_stdout
Definition: pe_types.h:110
void clone_append_meta(resource_t *rsc, xmlNode *xml)
enum pe_action_flags flags
Definition: pe_types.h:374
gboolean maintenance
Definition: pe_types.h:192
#define CRM_OP_PROBED
Definition: crm.h:127
#define pe_rsc_maintenance
Definition: pe_types.h:243
#define pe_rsc_failure_ignored
Definition: pe_types.h:242
void group_append_meta(resource_t *rsc, xmlNode *xml)
void clone_create_actions(resource_t *rsc, pe_working_set_t *data_set)
#define XML_CIB_TAG_STATUS
Definition: msg_xml.h:139
int custom_action_order(resource_t *lh_rsc, char *lh_task, action_t *lh_action, resource_t *rh_rsc, char *rh_task, action_t *rh_action, enum pe_ordering type, pe_working_set_t *data_set)
gboolean stonith_constraints(node_t *node, action_t *stonith_op, pe_working_set_t *data_set)
#define crm_log_xml_trace(xml, text)
Definition: logging.h:254
gboolean pe__is_remote_node(pe_node_t *node)
Definition: remote.c:36
gboolean crm_is_true(const char *s)
Definition: strings.c:172
#define XML_LRM_TAG_RSC_OP
Definition: msg_xml.h:228
#define pe_rsc_trace(rsc, fmt, args...)
Definition: internal.h:18
#define start_key(rsc)
Definition: internal.h:215
action_t * lh_action
Definition: internal.h:41
#define ID(x)
Definition: msg_xml.h:414
unsigned long long flags
Definition: pe_types.h:122
#define pe_err(fmt...)
Definition: internal.h:20
#define dump_node_scores(level, rsc, text, nodes)
Definition: internal.h:186
gboolean was_processing_error
Definition: common.c:18
int stickiness
Definition: pe_types.h:304
#define safe_str_eq(a, b)
Definition: util.h:59
int node_score_green
Definition: utils.c:64
gboolean order_actions(action_t *lh_action, action_t *rh_action, enum pe_ordering order)
Definition: utils.c:1776
op_digest_cache_t * rsc_action_digest_cmp(resource_t *rsc, xmlNode *xml_op, node_t *node, pe_working_set_t *data_set)
Definition: utils.c:2033
GList * pe__resource_actions(const pe_resource_t *rsc, const pe_node_t *node, const char *task, bool require_node)
Find all actions of given type for a resource.
Definition: utils.c:1517
gboolean was_processing_warning
Definition: common.c:19
gboolean probe_resources(pe_working_set_t *data_set)
gboolean unclean
Definition: pe_types.h:187
char * crm_strdup_printf(char const *format,...) __attribute__((__format__(__printf__
GList * GListPtr
Definition: crm.h:192
#define pe_flag_start_failure_fatal
Definition: pe_types.h:100
enum node_type type
Definition: pe_types.h:180
int node_score_yellow
Definition: utils.c:65
void pcmk__bundle_create_actions(pe_resource_t *rsc, pe_working_set_t *data_set)
#define crm_info(fmt, args...)
Definition: logging.h:243
#define pe_rsc_managed
Definition: pe_types.h:218
#define pe_rsc_orphan
Definition: pe_types.h:217
enum pe_graph_flags pcmk__multi_update_actions(pe_action_t *first, pe_action_t *then, pe_node_t *node, enum pe_action_flags flags, enum pe_action_flags filter, enum pe_ordering type, pe_working_set_t *data_set)
void pcmk__bundle_internal_constraints(pe_resource_t *rsc, pe_working_set_t *data_set)
char * generate_op_key(const char *rsc_id, const char *op_type, guint interval_ms)
Generate an operation key.
Definition: operations.c:39
void trigger_unfencing(resource_t *rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t *data_set)
Definition: utils.c:2275
void set_alloc_actions(pe_working_set_t *data_set)
void pcmk__bundle_append_meta(pe_resource_t *rsc, xmlNode *xml)
pe_ordering
Definition: pe_types.h:437
gboolean online
Definition: pe_types.h:183
uint64_t flags
Definition: remote.c:148
action_tasks
Definition: common.h:57
enum pe_action_flags native_action_flags(action_t *action, node_t *node)
enum crm_ais_msg_types type
Definition: internal.h:85
#define RSC_DEMOTE
Definition: crm.h:182
void pcmk__bundle_rsc_colocation_lh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
void native_rsc_colocation_rh(pe_resource_t *lh_rsc, pe_resource_t *rh_rsc, rsc_colocation_t *constraint, pe_working_set_t *data_set)
#define XML_AGENT_ATTR_CLASS
Definition: msg_xml.h:229
xmlNode * graph
Definition: pe_types.h:152
char * id
Definition: pe_types.h:284
GHashTable * allowed_nodes
Definition: pe_types.h:331
#define RSC_MIGRATED
Definition: crm.h:172
#define pe_flag_startup_probes
Definition: pe_types.h:104