root/lib/pacemaker/pcmk_sched_utils.c

/* */

DEFINITIONS

1. can_run_resources
2. pcmk__copy_node_table
3. pcmk__copy_node_list
4. sort_node_weight
5. sort_nodes_by_weight
6. native_deallocate
7. native_assign_node
8. log_action
9. can_run_any
10. create_pseudo_resource_op
11. pe_cancel_op
12. sched_shutdown_op
13. generate_transition_magic
14. append_digest
15. pcmk__create_history_xml
16. pcmk__new_logger
17. pcmk__threshold_reached
18. pcmk_free_injections

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 #include <crm/msg_xml.h>
  12 #include <crm/lrmd.h>       // lrmd_event_data_t
  13 #include <crm/common/xml_internal.h>
  14 #include <crm/lrmd_internal.h>
  15 #include <pacemaker-internal.h>
  16 #include <pacemaker.h>
  17 #include "libpacemaker_private.h"
  18 
  19 gboolean
  20 can_run_resources(const pe_node_t * node)
     /*  */
  21 {
  22     if (node == NULL) {
  23         return FALSE;
  24     }
  25 #if 0
  26     if (node->weight < 0) {
  27         return FALSE;
  28     }
  29 #endif
  30 
  31     if (node->details->online == FALSE
  32         || node->details->shutdown || node->details->unclean
  33         || node->details->standby || node->details->maintenance) {
  34         crm_trace("%s: online=%d, unclean=%d, standby=%d, maintenance=%d",
  35                   node->details->uname, node->details->online,
  36                   node->details->unclean, node->details->standby, node->details->maintenance);
  37         return FALSE;
  38     }
  39     return TRUE;
  40 }
  41 
  42 /*!
  43  * \internal
  44  * \brief Copy a hash table of node objects
  45  *
  46  * \param[in] nodes  Hash table to copy
  47  *
  48  * \return New copy of nodes (or NULL if nodes is NULL)
  49  */
  50 GHashTable *
  51 pcmk__copy_node_table(GHashTable *nodes)
     /*  */
  52 {
  53     GHashTable *new_table = NULL;
  54     GHashTableIter iter;
  55     pe_node_t *node = NULL;
  56 
  57     if (nodes == NULL) {
  58         return NULL;
  59     }
  60     new_table = pcmk__strkey_table(NULL, free);
  61     g_hash_table_iter_init(&iter, nodes);
  62     while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
  63         pe_node_t *new_node = pe__copy_node(node);
  64 
  65         g_hash_table_insert(new_table, (gpointer) new_node->details->id,
  66                             new_node);
  67     }
  68     return new_table;
  69 }
  70 
  71 /*!
  72  * \internal
  73  * \brief Copy a list of node objects
  74  *
  75  * \param[in] list   List to copy
  76  * \param[in] reset  Set copies' scores to 0
  77  *
  78  * \return New list of shallow copies of nodes in original list
  79  */
  80 GList *
  81 pcmk__copy_node_list(const GList *list, bool reset)
     /*  */
  82 {
  83     GList *result = NULL;
  84 
  85     for (const GList *gIter = list; gIter != NULL; gIter = gIter->next) {
  86         pe_node_t *new_node = NULL;
  87         pe_node_t *this_node = (pe_node_t *) gIter->data;
  88 
  89         new_node = pe__copy_node(this_node);
  90         if (reset) {
  91             new_node->weight = 0;
  92         }
  93         result = g_list_prepend(result, new_node);
  94     }
  95     return result;
  96 }
  97 
  98 struct node_weight_s {
  99     pe_node_t *active;
 100     pe_working_set_t *data_set;
 101 };
 102 
 103 /* return -1 if 'a' is more preferred
 104  * return  1 if 'b' is more preferred
 105  */
 106 
 107 static gint
 108 sort_node_weight(gconstpointer a, gconstpointer b, gpointer data)
     /*  */
 109 {
 110     const pe_node_t *node1 = (const pe_node_t *)a;
 111     const pe_node_t *node2 = (const pe_node_t *)b;
 112     struct node_weight_s *nw = data;
 113 
 114     int node1_weight = 0;
 115     int node2_weight = 0;
 116 
 117     int result = 0;
 118 
 119     if (a == NULL) {
 120         return 1;
 121     }
 122     if (b == NULL) {
 123         return -1;
 124     }
 125 
 126     node1_weight = node1->weight;
 127     node2_weight = node2->weight;
 128 
 129     if (can_run_resources(node1) == FALSE) {
 130         node1_weight = -INFINITY;
 131     }
 132     if (can_run_resources(node2) == FALSE) {
 133         node2_weight = -INFINITY;
 134     }
 135 
 136     if (node1_weight > node2_weight) {
 137         crm_trace("%s (%d) > %s (%d) : weight",
 138                   node1->details->uname, node1_weight, node2->details->uname, node2_weight);
 139         return -1;
 140     }
 141 
 142     if (node1_weight < node2_weight) {
 143         crm_trace("%s (%d) < %s (%d) : weight",
 144                   node1->details->uname, node1_weight, node2->details->uname, node2_weight);
 145         return 1;
 146     }
 147 
 148     crm_trace("%s (%d) == %s (%d) : weight",
 149               node1->details->uname, node1_weight, node2->details->uname, node2_weight);
 150 
 151     if (pcmk__str_eq(nw->data_set->placement_strategy, "minimal", pcmk__str_casei)) {
 152         goto equal;
 153     }
 154 
 155     if (pcmk__str_eq(nw->data_set->placement_strategy, "balanced", pcmk__str_casei)) {
 156         result = compare_capacity(node1, node2);
 157         if (result < 0) {
 158             crm_trace("%s > %s : capacity (%d)",
 159                       node1->details->uname, node2->details->uname, result);
 160             return -1;
 161         } else if (result > 0) {
 162             crm_trace("%s < %s : capacity (%d)",
 163                       node1->details->uname, node2->details->uname, result);
 164             return 1;
 165         }
 166     }
 167 
 168     /* now try to balance resources across the cluster */
 169     if (node1->details->num_resources < node2->details->num_resources) {
 170         crm_trace("%s (%d) > %s (%d) : resources",
 171                   node1->details->uname, node1->details->num_resources,
 172                   node2->details->uname, node2->details->num_resources);
 173         return -1;
 174 
 175     } else if (node1->details->num_resources > node2->details->num_resources) {
 176         crm_trace("%s (%d) < %s (%d) : resources",
 177                   node1->details->uname, node1->details->num_resources,
 178                   node2->details->uname, node2->details->num_resources);
 179         return 1;
 180     }
 181 
 182     if (nw->active && nw->active->details == node1->details) {
 183         crm_trace("%s (%d) > %s (%d) : active",
 184                   node1->details->uname, node1->details->num_resources,
 185                   node2->details->uname, node2->details->num_resources);
 186         return -1;
 187     } else if (nw->active && nw->active->details == node2->details) {
 188         crm_trace("%s (%d) < %s (%d) : active",
 189                   node1->details->uname, node1->details->num_resources,
 190                   node2->details->uname, node2->details->num_resources);
 191         return 1;
 192     }
 193   equal:
 194     crm_trace("%s = %s", node1->details->uname, node2->details->uname);
 195     return strcmp(node1->details->uname, node2->details->uname);
 196 }
 197 
 198 GList *
 199 sort_nodes_by_weight(GList *nodes, pe_node_t *active_node,
     /*  */
 200                      pe_working_set_t *data_set)
 201 {
 202     struct node_weight_s nw = { active_node, data_set };
 203 
 204     return g_list_sort_with_data(nodes, sort_node_weight, &nw);
 205 }
 206 
 207 void
 208 native_deallocate(pe_resource_t * rsc)
     /*  */
 209 {
 210     if (rsc->allocated_to) {
 211         pe_node_t *old = rsc->allocated_to;
 212 
 213         crm_info("Deallocating %s from %s", rsc->id, old->details->uname);
 214         pe__set_resource_flags(rsc, pe_rsc_provisional);
 215         rsc->allocated_to = NULL;
 216 
 217         old->details->allocated_rsc = g_list_remove(old->details->allocated_rsc, rsc);
 218         old->details->num_resources--;
 219         /* old->count--; */
 220         calculate_utilization(old->details->utilization, rsc->utilization, TRUE);
 221         free(old);
 222     }
 223 }
 224 
 225 gboolean
 226 native_assign_node(pe_resource_t *rsc, pe_node_t *chosen, gboolean force)
     /*  */
 227 {
 228     pcmk__output_t *out = rsc->cluster->priv;
 229 
 230     CRM_ASSERT(rsc->variant == pe_native);
 231 
 232     if (force == FALSE && chosen != NULL) {
 233         bool unset = FALSE;
 234 
 235         if(chosen->weight < 0) {
 236             unset = TRUE;
 237 
 238             // Allow the graph to assume that the remote resource will come up
 239         } else if (!can_run_resources(chosen) && !pe__is_guest_node(chosen)) {
 240             unset = TRUE;
 241         }
 242 
 243         if(unset) {
 244             crm_debug("All nodes for resource %s are unavailable"
 245                       ", unclean or shutting down (%s: %d, %d)",
 246                       rsc->id, chosen->details->uname, can_run_resources(chosen), chosen->weight);
 247             pe__set_next_role(rsc, RSC_ROLE_STOPPED, "node availability");
 248             chosen = NULL;
 249         }
 250     }
 251 
 252     /* todo: update the old node for each resource to reflect its
 253      * new resource count
 254      */
 255 
 256     native_deallocate(rsc);
 257     pe__clear_resource_flags(rsc, pe_rsc_provisional);
 258 
 259     if (chosen == NULL) {
 260         GList *gIter = NULL;
 261         char *rc_inactive = pcmk__itoa(PCMK_OCF_NOT_RUNNING);
 262 
 263         crm_debug("Could not allocate a node for %s", rsc->id);
 264         pe__set_next_role(rsc, RSC_ROLE_STOPPED, "unable to allocate");
 265 
 266         for (gIter = rsc->actions; gIter != NULL; gIter = gIter->next) {
 267             pe_action_t *op = (pe_action_t *) gIter->data;
 268             const char *interval_ms_s = g_hash_table_lookup(op->meta, XML_LRM_ATTR_INTERVAL_MS);
 269 
 270             crm_debug("Processing %s", op->uuid);
 271             if(pcmk__str_eq(RSC_STOP, op->task, pcmk__str_casei)) {
 272                 pe__clear_action_flags(op, pe_action_optional);
 273 
 274             } else if(pcmk__str_eq(RSC_START, op->task, pcmk__str_casei)) {
 275                 pe__clear_action_flags(op, pe_action_runnable);
 276                 //pe__set_resource_flags(rsc, pe_rsc_block);
 277 
 278             } else if (interval_ms_s && !pcmk__str_eq(interval_ms_s, "0", pcmk__str_casei)) {
 279                 if(pcmk__str_eq(rc_inactive, g_hash_table_lookup(op->meta, XML_ATTR_TE_TARGET_RC), pcmk__str_casei)) {
 280                     /* This is a recurring monitor for the stopped state, leave it alone */
 281 
 282                 } else {
 283                     /* Normal monitor operation, cancel it */
 284                     pe__clear_action_flags(op, pe_action_runnable);
 285                 }
 286             }
 287         }
 288 
 289         free(rc_inactive);
 290         return FALSE;
 291     }
 292 
 293     crm_debug("Assigning %s to %s", chosen->details->uname, rsc->id);
 294     rsc->allocated_to = pe__copy_node(chosen);
 295 
 296     chosen->details->allocated_rsc = g_list_prepend(chosen->details->allocated_rsc, rsc);
 297     chosen->details->num_resources++;
 298     chosen->count++;
 299     calculate_utilization(chosen->details->utilization, rsc->utilization, FALSE);
 300 
 301     if (pcmk_is_set(rsc->cluster->flags, pe_flag_show_utilization)) {
 302         out->message(out, "resource-util", rsc, chosen, __func__);
 303     }
 304 
 305     return TRUE;
 306 }
 307 
 308 void
 309 log_action(unsigned int log_level, const char *pre_text, pe_action_t * action, gboolean details)
     /*  */
 310 {
 311     const char *node_uname = NULL;
 312     const char *node_uuid = NULL;
 313     const char *desc = NULL;
 314 
 315     if (action == NULL) {
 316         crm_trace("%s%s: <NULL>", pre_text == NULL ? "" : pre_text, pre_text == NULL ? "" : ": ");
 317         return;
 318     }
 319 
 320     if (pcmk_is_set(action->flags, pe_action_pseudo)) {
 321         node_uname = NULL;
 322         node_uuid = NULL;
 323 
 324     } else if (action->node != NULL) {
 325         node_uname = action->node->details->uname;
 326         node_uuid = action->node->details->id;
 327     } else {
 328         node_uname = "<none>";
 329         node_uuid = NULL;
 330     }
 331 
 332     switch (text2task(action->task)) {
 333         case stonith_node:
 334         case shutdown_crm:
 335             if (pcmk_is_set(action->flags, pe_action_pseudo)) {
 336                 desc = "Pseudo ";
 337             } else if (pcmk_is_set(action->flags, pe_action_optional)) {
 338                 desc = "Optional ";
 339             } else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
 340                 desc = "!!Non-Startable!! ";
 341             } else if (pcmk_is_set(action->flags, pe_action_processed)) {
 342                desc = "";
 343             } else {
 344                desc = "(Provisional) ";
 345             }
 346             crm_trace("%s%s%sAction %d: %s%s%s%s%s%s",
 347                       ((pre_text == NULL)? "" : pre_text),
 348                       ((pre_text == NULL)? "" : ": "),
 349                       desc, action->id, action->uuid,
 350                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
 351                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
 352                       (node_uuid? ")" : ""));
 353             break;
 354         default:
 355             if (pcmk_is_set(action->flags, pe_action_optional)) {
 356                 desc = "Optional ";
 357             } else if (pcmk_is_set(action->flags, pe_action_pseudo)) {
 358                 desc = "Pseudo ";
 359             } else if (!pcmk_is_set(action->flags, pe_action_runnable)) {
 360                 desc = "!!Non-Startable!! ";
 361             } else if (pcmk_is_set(action->flags, pe_action_processed)) {
 362                desc = "";
 363             } else {
 364                desc = "(Provisional) ";
 365             }
 366             crm_trace("%s%s%sAction %d: %s %s%s%s%s%s%s",
 367                       ((pre_text == NULL)? "" : pre_text),
 368                       ((pre_text == NULL)? "" : ": "),
 369                       desc, action->id, action->uuid,
 370                       (action->rsc? action->rsc->id : "<none>"),
 371                       (node_uname? "\ton " : ""), (node_uname? node_uname : ""),
 372                       (node_uuid? "\t\t(" : ""), (node_uuid? node_uuid : ""),
 373                       (node_uuid? ")" : ""));
 374             break;
 375     }
 376 
 377     if (details) {
 378         GList *gIter = NULL;
 379 
 380         crm_trace("\t\t====== Preceding Actions");
 381 
 382         gIter = action->actions_before;
 383         for (; gIter != NULL; gIter = gIter->next) {
 384             pe_action_wrapper_t *other = (pe_action_wrapper_t *) gIter->data;
 385 
 386             log_action(log_level + 1, "\t\t", other->action, FALSE);
 387         }
 388 
 389         crm_trace("\t\t====== Subsequent Actions");
 390 
 391         gIter = action->actions_after;
 392         for (; gIter != NULL; gIter = gIter->next) {
 393             pe_action_wrapper_t *other = (pe_action_wrapper_t *) gIter->data;
 394 
 395             log_action(log_level + 1, "\t\t", other->action, FALSE);
 396         }
 397 
 398         crm_trace("\t\t====== End");
 399 
 400     } else {
 401         crm_trace("\t\t(before=%d, after=%d)",
 402                   g_list_length(action->actions_before), g_list_length(action->actions_after));
 403     }
 404 }
 405 
 406 gboolean
 407 can_run_any(GHashTable * nodes)
     /*  */
 408 {
 409     GHashTableIter iter;
 410     pe_node_t *node = NULL;
 411 
 412     if (nodes == NULL) {
 413         return FALSE;
 414     }
 415 
 416     g_hash_table_iter_init(&iter, nodes);
 417     while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
 418         if (can_run_resources(node) && node->weight >= 0) {
 419             return TRUE;
 420         }
 421     }
 422 
 423     return FALSE;
 424 }
 425 
 426 pe_action_t *
 427 create_pseudo_resource_op(pe_resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set)
     /*  */
 428 {
 429     pe_action_t *action = custom_action(rsc, pcmk__op_key(rsc->id, task, 0),
 430                                         task, NULL, optional, TRUE, data_set);
 431 
 432     pe__set_action_flags(action, pe_action_pseudo);
 433     if(runnable) {
 434         pe__set_action_flags(action, pe_action_runnable);
 435     }
 436     return action;
 437 }
 438 
 439 /*!
 440  * \internal
 441  * \brief Create an executor cancel op
 442  *
 443  * \param[in] rsc          Resource of action to cancel
 444  * \param[in] task         Name of action to cancel
 445  * \param[in] interval_ms  Interval of action to cancel
 446  * \param[in] node         Node of action to cancel
 447  * \param[in] data_set     Working set of cluster
 448  *
 449  * \return Created op
 450  */
 451 pe_action_t *
 452 pe_cancel_op(pe_resource_t *rsc, const char *task, guint interval_ms,
     /*  */
 453              pe_node_t *node, pe_working_set_t *data_set)
 454 {
 455     pe_action_t *cancel_op;
 456     char *interval_ms_s = crm_strdup_printf("%u", interval_ms);
 457 
 458     // @TODO dangerous if possible to schedule another action with this key
 459     char *key = pcmk__op_key(rsc->id, task, interval_ms);
 460 
 461     cancel_op = custom_action(rsc, key, RSC_CANCEL, node, FALSE, TRUE,
 462                               data_set);
 463 
 464     free(cancel_op->task);
 465     cancel_op->task = strdup(RSC_CANCEL);
 466 
 467     free(cancel_op->cancel_task);
 468     cancel_op->cancel_task = strdup(task);
 469 
 470     add_hash_param(cancel_op->meta, XML_LRM_ATTR_TASK, task);
 471     add_hash_param(cancel_op->meta, XML_LRM_ATTR_INTERVAL_MS, interval_ms_s);
 472     free(interval_ms_s);
 473 
 474     return cancel_op;
 475 }
 476 
 477 /*!
 478  * \internal
 479  * \brief Create a shutdown op for a scheduler transition
 480  *
 481  * \param[in] node         Node being shut down
 482  * \param[in] data_set     Working set of cluster
 483  *
 484  * \return Created op
 485  */
 486 pe_action_t *
 487 sched_shutdown_op(pe_node_t *node, pe_working_set_t *data_set)
     /*  */
 488 {
 489     char *shutdown_id = crm_strdup_printf("%s-%s", CRM_OP_SHUTDOWN,
 490                                           node->details->uname);
 491 
 492     pe_action_t *shutdown_op = custom_action(NULL, shutdown_id, CRM_OP_SHUTDOWN,
 493                                              node, FALSE, TRUE, data_set);
 494 
 495     pcmk__order_stops_before_shutdown(node, shutdown_op, data_set);
 496     add_hash_param(shutdown_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE);
 497     return shutdown_op;
 498 }
 499 
 500 static char *
 501 generate_transition_magic(const char *transition_key, int op_status, int op_rc)
     /*  */
 502 {
 503     CRM_CHECK(transition_key != NULL, return NULL);
 504     return crm_strdup_printf("%d:%d;%s", op_status, op_rc, transition_key);
 505 }
 506 
 507 static void
 508 append_digest(lrmd_event_data_t *op, xmlNode *update, const char *version,
     /*  */
 509               const char *magic, int level)
 510 {
 511     /* this will enable us to later determine that the
 512      *   resource's parameters have changed and we should force
 513      *   a restart
 514      */
 515     char *digest = NULL;
 516     xmlNode *args_xml = NULL;
 517 
 518     if (op->params == NULL) {
 519         return;
 520     }
 521 
 522     args_xml = create_xml_node(NULL, XML_TAG_PARAMS);
 523     g_hash_table_foreach(op->params, hash2field, args_xml);
 524     pcmk__filter_op_for_digest(args_xml);
 525     digest = calculate_operation_digest(args_xml, version);
 526 
 527 #if 0
 528     if (level < get_crm_log_level()
 529         && op->interval_ms == 0 && pcmk__str_eq(op->op_type, CRMD_ACTION_START, pcmk__str_none)) {
 530         char *digest_source = dump_xml_unformatted(args_xml);
 531 
 532         do_crm_log(level, "Calculated digest %s for %s (%s). Source: %s\n",
 533                    digest, ID(update), magic, digest_source);
 534         free(digest_source);
 535     }
 536 #endif
 537     crm_xml_add(update, XML_LRM_ATTR_OP_DIGEST, digest);
 538 
 539     free_xml(args_xml);
 540     free(digest);
 541 }
 542 
 543 #define FAKE_TE_ID     "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
 544 
 545 /*!
 546  * \internal
 547  * \brief Create XML for resource operation history update
 548  *
 549  * \param[in,out] parent          Parent XML node to add to
 550  * \param[in,out] op              Operation event data
 551  * \param[in]     caller_version  DC feature set
 552  * \param[in]     target_rc       Expected result of operation
 553  * \param[in]     node            Name of node on which operation was performed
 554  * \param[in]     origin          Arbitrary description of update source
 555  * \param[in]     level           A log message will be logged at this level
 556  *
 557  * \return Newly created XML node for history update
 558  */
 559 xmlNode *
 560 pcmk__create_history_xml(xmlNode *parent, lrmd_event_data_t *op,
     /*  */
 561                          const char *caller_version, int target_rc,
 562                          const char *node, const char *origin, int level)
 563 {
 564     char *key = NULL;
 565     char *magic = NULL;
 566     char *op_id = NULL;
 567     char *op_id_additional = NULL;
 568     char *local_user_data = NULL;
 569     const char *exit_reason = NULL;
 570 
 571     xmlNode *xml_op = NULL;
 572     const char *task = NULL;
 573 
 574     CRM_CHECK(op != NULL, return NULL);
 575     do_crm_log(level, "%s: Updating resource %s after %s op %s (interval=%u)",
 576                origin, op->rsc_id, op->op_type,
 577                pcmk_exec_status_str(op->op_status), op->interval_ms);
 578 
 579     crm_trace("DC version: %s", caller_version);
 580 
 581     task = op->op_type;
 582 
 583     /* Record a successful agent reload as a start, and a failed one as a
 584      * monitor, to make life easier for the scheduler when determining the
 585      * current state.
 586      *
 587      * @COMPAT We should check "reload" here only if the operation was for a
 588      * pre-OCF-1.1 resource agent, but we don't know that here, and we should
 589      * only ever get results for actions scheduled by us, so we can reasonably
 590      * assume any "reload" is actually a pre-1.1 agent reload.
 591      */
 592     if (pcmk__str_any_of(task, CRMD_ACTION_RELOAD, CRMD_ACTION_RELOAD_AGENT,
 593                          NULL)) {
 594         if (op->op_status == PCMK_EXEC_DONE) {
 595             task = CRMD_ACTION_START;
 596         } else {
 597             task = CRMD_ACTION_STATUS;
 598         }
 599     }
 600 
 601     key = pcmk__op_key(op->rsc_id, task, op->interval_ms);
 602     if (pcmk__str_eq(task, CRMD_ACTION_NOTIFY, pcmk__str_none)) {
 603         const char *n_type = crm_meta_value(op->params, "notify_type");
 604         const char *n_task = crm_meta_value(op->params, "notify_operation");
 605 
 606         CRM_LOG_ASSERT(n_type != NULL);
 607         CRM_LOG_ASSERT(n_task != NULL);
 608         op_id = pcmk__notify_key(op->rsc_id, n_type, n_task);
 609 
 610         if (op->op_status != PCMK_EXEC_PENDING) {
 611             /* Ignore notify errors.
 612              *
 613              * @TODO It might be better to keep the correct result here, and
 614              * ignore it in process_graph_event().
 615              */
 616             lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 617         }
 618 
 619     } else if (did_rsc_op_fail(op, target_rc)) {
 620         op_id = pcmk__op_key(op->rsc_id, "last_failure", 0);
 621         if (op->interval_ms == 0) {
 622             // Ensure 'last' gets updated, in case record-pending is true
 623             op_id_additional = pcmk__op_key(op->rsc_id, "last", 0);
 624         }
 625         exit_reason = op->exit_reason;
 626 
 627     } else if (op->interval_ms > 0) {
 628         op_id = strdup(key);
 629 
 630     } else {
 631         op_id = pcmk__op_key(op->rsc_id, "last", 0);
 632     }
 633 
 634   again:
 635     xml_op = pcmk__xe_match(parent, XML_LRM_TAG_RSC_OP, XML_ATTR_ID, op_id);
 636     if (xml_op == NULL) {
 637         xml_op = create_xml_node(parent, XML_LRM_TAG_RSC_OP);
 638     }
 639 
 640     if (op->user_data == NULL) {
 641         crm_debug("Generating fake transition key for: " PCMK__OP_FMT
 642                   " %d from %s", op->rsc_id, op->op_type, op->interval_ms,
 643                   op->call_id, origin);
 644         local_user_data = pcmk__transition_key(-1, op->call_id, target_rc,
 645                                                FAKE_TE_ID);
 646         op->user_data = local_user_data;
 647     }
 648 
 649     if(magic == NULL) {
 650         magic = generate_transition_magic(op->user_data, op->op_status, op->rc);
 651     }
 652 
 653     crm_xml_add(xml_op, XML_ATTR_ID, op_id);
 654     crm_xml_add(xml_op, XML_LRM_ATTR_TASK_KEY, key);
 655     crm_xml_add(xml_op, XML_LRM_ATTR_TASK, task);
 656     crm_xml_add(xml_op, XML_ATTR_ORIGIN, origin);
 657     crm_xml_add(xml_op, XML_ATTR_CRM_VERSION, caller_version);
 658     crm_xml_add(xml_op, XML_ATTR_TRANSITION_KEY, op->user_data);
 659     crm_xml_add(xml_op, XML_ATTR_TRANSITION_MAGIC, magic);
 660     crm_xml_add(xml_op, XML_LRM_ATTR_EXIT_REASON, exit_reason == NULL ? "" : exit_reason);
 661     crm_xml_add(xml_op, XML_LRM_ATTR_TARGET, node); /* For context during triage */
 662 
 663     crm_xml_add_int(xml_op, XML_LRM_ATTR_CALLID, op->call_id);
 664     crm_xml_add_int(xml_op, XML_LRM_ATTR_RC, op->rc);
 665     crm_xml_add_int(xml_op, XML_LRM_ATTR_OPSTATUS, op->op_status);
 666     crm_xml_add_ms(xml_op, XML_LRM_ATTR_INTERVAL_MS, op->interval_ms);
 667 
 668     if (compare_version("2.1", caller_version) <= 0) {
 669         if (op->t_run || op->t_rcchange || op->exec_time || op->queue_time) {
 670             crm_trace("Timing data (" PCMK__OP_FMT
 671                       "): last=%u change=%u exec=%u queue=%u",
 672                       op->rsc_id, op->op_type, op->interval_ms,
 673                       op->t_run, op->t_rcchange, op->exec_time, op->queue_time);
 674 
 675             if ((op->interval_ms != 0) && (op->t_rcchange != 0)) {
 676                 // Recurring ops may have changed rc after initial run
 677                 crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
 678                                (long long) op->t_rcchange);
 679             } else {
 680                 crm_xml_add_ll(xml_op, XML_RSC_OP_LAST_CHANGE,
 681                                (long long) op->t_run);
 682             }
 683 
 684             crm_xml_add_int(xml_op, XML_RSC_OP_T_EXEC, op->exec_time);
 685             crm_xml_add_int(xml_op, XML_RSC_OP_T_QUEUE, op->queue_time);
 686         }
 687     }
 688 
 689     if (pcmk__str_any_of(op->op_type, CRMD_ACTION_MIGRATE, CRMD_ACTION_MIGRATED, NULL)) {
 690         /*
 691          * Record migrate_source and migrate_target always for migrate ops.
 692          */
 693         const char *name = XML_LRM_ATTR_MIGRATE_SOURCE;
 694 
 695         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 696 
 697         name = XML_LRM_ATTR_MIGRATE_TARGET;
 698         crm_xml_add(xml_op, name, crm_meta_value(op->params, name));
 699     }
 700 
 701     append_digest(op, xml_op, caller_version, magic, LOG_DEBUG);
 702 
 703     if (op_id_additional) {
 704         free(op_id);
 705         op_id = op_id_additional;
 706         op_id_additional = NULL;
 707         goto again;
 708     }
 709 
 710     if (local_user_data) {
 711         free(local_user_data);
 712         op->user_data = NULL;
 713     }
 714     free(magic);
 715     free(op_id);
 716     free(key);
 717     return xml_op;
 718 }
 719 
 720 pcmk__output_t *
 721 pcmk__new_logger(void)
     /*  */
 722 {
 723     int rc = pcmk_rc_ok;
 724     pcmk__output_t *out = NULL;
 725     const char* argv[] = { "", NULL };
 726     pcmk__supported_format_t formats[] = {
 727         PCMK__SUPPORTED_FORMAT_LOG,
 728         { NULL, NULL, NULL }
 729     };
 730 
 731     pcmk__register_formats(NULL, formats);
 732     rc = pcmk__output_new(&out, "log", NULL, (char**)argv);
 733     if ((rc != pcmk_rc_ok) || (out == NULL)) {
 734         crm_err("Can't log resource details due to internal error: %s\n",
 735                 pcmk_rc_str(rc));
 736         return NULL;
 737     }
 738 
 739     pe__register_messages(out);
 740     pcmk__register_lib_messages(out);
 741     return out;
 742 }
 743 
 744 /*!
 745  * \internal
 746  * \brief Check whether a resource has reached its migration threshold on a node
 747  *
 748  * \param[in]  rsc       Resource to check
 749  * \param[in]  node      Node to check
 750  * \param[in]  data_set  Cluster working set
 751  * \param[out] failed    If the threshold has been reached, this will be set to
 752  *                       the resource that failed (possibly a parent of \p rsc)
 753  *
 754  * \return true if the migration threshold has been reached, false otherwise
 755  */
 756 bool
 757 pcmk__threshold_reached(pe_resource_t *rsc, pe_node_t *node,
     /*  */
 758                         pe_working_set_t *data_set, pe_resource_t **failed)
 759 {
 760     int fail_count, remaining_tries;
 761     pe_resource_t *rsc_to_ban = rsc;
 762 
 763     // Migration threshold of 0 means never force away
 764     if (rsc->migration_threshold == 0) {
 765         return false;
 766     }
 767 
 768     // If we're ignoring failures, also ignore the migration threshold
 769     if (pcmk_is_set(rsc->flags, pe_rsc_failure_ignored)) {
 770         return false;
 771     }
 772 
 773     // If there are no failures, there's no need to force away
 774     fail_count = pe_get_failcount(node, rsc, NULL,
 775                                   pe_fc_effective|pe_fc_fillers, NULL,
 776                                   data_set);
 777     if (fail_count <= 0) {
 778         return false;
 779     }
 780 
 781     // If failed resource is anonymous clone instance, we'll force clone away
 782     if (!pcmk_is_set(rsc->flags, pe_rsc_unique)) {
 783         rsc_to_ban = uber_parent(rsc);
 784     }
 785 
 786     // How many more times recovery will be tried on this node
 787     remaining_tries = rsc->migration_threshold - fail_count;
 788 
 789     if (remaining_tries <= 0) {
 790         crm_warn("%s cannot run on %s due to reaching migration threshold "
 791                  "(clean up resource to allow again)"
 792                  CRM_XS " failures=%d migration-threshold=%d",
 793                  rsc_to_ban->id, node->details->uname, fail_count,
 794                  rsc->migration_threshold);
 795         if (failed != NULL) {
 796             *failed = rsc_to_ban;
 797         }
 798         return true;
 799     }
 800 
 801     crm_info("%s can fail %d more time%s on "
 802              "%s before reaching migration threshold (%d)",
 803              rsc_to_ban->id, remaining_tries, pcmk__plural_s(remaining_tries),
 804              node->details->uname, rsc->migration_threshold);
 805     return false;
 806 }
 807 
 808 void
 809 pcmk_free_injections(pcmk_injections_t *injections)
     /*  */
 810 {
 811     if (injections == NULL) {
 812         return;
 813     }
 814 
 815     g_list_free_full(injections->node_up, g_free);
 816     g_list_free_full(injections->node_down, g_free);
 817     g_list_free_full(injections->node_fail, g_free);
 818     g_list_free_full(injections->op_fail, g_free);
 819     g_list_free_full(injections->op_inject, g_free);
 820     g_list_free_full(injections->ticket_grant, g_free);
 821     g_list_free_full(injections->ticket_revoke, g_free);
 822     g_list_free_full(injections->ticket_standby, g_free);
 823     g_list_free_full(injections->ticket_activate, g_free);
 824     free(injections->quorum);
 825     free(injections->watchdog);
 826 
 827     free(injections);
 828 }

/* */