root/daemons/controld/controld_te_callbacks.c

/* */

DEFINITIONS

1. shutdown_lock_cleared
2. te_update_diff_v1
3. process_lrm_resource_diff
4. process_resource_updates
5. extract_node_uuid
6. abort_unless_down
7. process_op_deletion
8. process_delete_diff
9. process_node_state_diff
10. process_status_diff
11. process_cib_diff
12. te_update_diff_element_v2
13. te_update_diff_v2
14. te_update_diff
15. process_te_message
16. cib_action_updated
17. action_timer_callback

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 
  18 #include <pacemaker-controld.h>
  19 
  20 void te_update_confirm(const char *event, xmlNode * msg);
  21 
  22 #define RSC_OP_PREFIX "//" PCMK__XE_DIFF_ADDED "//" PCMK_XE_CIB \
  23                       "//" PCMK__XE_LRM_RSC_OP "[@" PCMK_XA_ID "='"
  24 
  25 // An explicit PCMK_OPT_SHUTDOWN_LOCK of 0 means the lock has been cleared
  26 static bool
  27 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  28 {
  29     time_t shutdown_lock = 0;
  30 
  31     return (crm_element_value_epoch(lrm_resource, PCMK_OPT_SHUTDOWN_LOCK,
  32                                     &shutdown_lock) == pcmk_ok)
  33            && (shutdown_lock == 0);
  34 }
  35 
  36 static void
  37 te_update_diff_v1(const char *event, xmlNode *diff)
     /*  */
  38 {
  39     int lpc, max;
  40     xmlXPathObject *xpathObj = NULL;
  41     GString *rsc_op_xpath = NULL;
  42 
  43     CRM_CHECK(diff != NULL, return);
  44 
  45     pcmk__output_set_log_level(controld_globals.logger_out, LOG_TRACE);
  46     controld_globals.logger_out->message(controld_globals.logger_out,
  47                                          "xml-patchset", diff);
  48 
  49     if (cib__config_changed_v1(NULL, NULL, &diff)) {
  50         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  51                          "Non-status change", diff);
  52         goto bail;              /* configuration changed */
  53     }
  54 
  55     /* Tickets Attributes - Added/Updated */
  56     xpathObj =
  57         xpath_search(diff,
  58                      "//" PCMK__XE_CIB_UPDATE_RESULT
  59                      "//" PCMK__XE_DIFF_ADDED
  60                      "//" PCMK_XE_TICKETS);
  61     if (numXpathResults(xpathObj) > 0) {
  62         xmlNode *aborted = getXpathResult(xpathObj, 0);
  63 
  64         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  65                          "Ticket attribute: update", aborted);
  66         goto bail;
  67 
  68     }
  69     freeXpathObject(xpathObj);
  70 
  71     /* Tickets Attributes - Removed */
  72     xpathObj =
  73         xpath_search(diff,
  74                      "//" PCMK__XE_CIB_UPDATE_RESULT
  75                      "//" PCMK__XE_DIFF_REMOVED
  76                      "//" PCMK_XE_TICKETS);
  77     if (numXpathResults(xpathObj) > 0) {
  78         xmlNode *aborted = getXpathResult(xpathObj, 0);
  79 
  80         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  81                          "Ticket attribute: removal", aborted);
  82         goto bail;
  83     }
  84     freeXpathObject(xpathObj);
  85 
  86     /* Transient Attributes - Removed */
  87     xpathObj =
  88         xpath_search(diff,
  89                      "//" PCMK__XE_CIB_UPDATE_RESULT
  90                      "//" PCMK__XE_DIFF_REMOVED
  91                      "//" PCMK__XE_TRANSIENT_ATTRIBUTES);
  92     if (numXpathResults(xpathObj) > 0) {
  93         xmlNode *aborted = getXpathResult(xpathObj, 0);
  94 
  95         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  96                          "Transient attribute: removal", aborted);
  97         goto bail;
  98 
  99     }
 100     freeXpathObject(xpathObj);
 101 
 102     // Check for PCMK__XE_LRM_RESOURCE entries
 103     xpathObj = xpath_search(diff,
 104                             "//" PCMK__XE_CIB_UPDATE_RESULT
 105                             "//" PCMK__XE_DIFF_ADDED
 106                             "//" PCMK__XE_LRM_RESOURCE);
 107     max = numXpathResults(xpathObj);
 108 
 109     /*
 110      * Updates by, or in response to, graph actions will never affect more than
 111      * one resource at a time, so such updates indicate an LRM refresh. In that
 112      * case, start a new transition rather than check each result individually,
 113      * which can result in _huge_ speedups in large clusters.
 114      *
 115      * Unfortunately, we can only do so when there are no pending actions.
 116      * Otherwise, we could mistakenly throw away those results here, and
 117      * the cluster will stall waiting for them and time out the operation.
 118      */
 119     if ((controld_globals.transition_graph->pending == 0) && (max > 1)) {
 120         crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
 121                   max);
 122         crm_log_xml_trace(diff, "lrm-refresh");
 123         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 124                          "History refresh", NULL);
 125         goto bail;
 126     }
 127 
 128     if (max == 1) {
 129         xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
 130 
 131         if (shutdown_lock_cleared(lrm_resource)) {
 132             // @TODO would be more efficient to abort once after transition done
 133             abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 134                              "Shutdown lock cleared", lrm_resource);
 135             // Still process results, so we stop timers and update failcounts
 136         }
 137     }
 138     freeXpathObject(xpathObj);
 139 
 140     /* Process operation updates */
 141     xpathObj =
 142         xpath_search(diff,
 143                      "//" PCMK__XE_CIB_UPDATE_RESULT
 144                      "//" PCMK__XE_DIFF_ADDED
 145                      "//" PCMK__XE_LRM_RSC_OP);
 146     max = numXpathResults(xpathObj);
 147     if (max > 0) {
 148         int lpc = 0;
 149 
 150         for (lpc = 0; lpc < max; lpc++) {
 151             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 152             const char *node = get_node_id(rsc_op);
 153 
 154             process_graph_event(rsc_op, node);
 155         }
 156     }
 157     freeXpathObject(xpathObj);
 158 
 159     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
 160     xpathObj = xpath_search(diff,
 161                             "//" PCMK__XE_DIFF_REMOVED
 162                             "//" PCMK__XE_LRM_RSC_OP);
 163     max = numXpathResults(xpathObj);
 164     for (lpc = 0; lpc < max; lpc++) {
 165         const char *op_id = NULL;
 166         xmlXPathObject *op_match = NULL;
 167         xmlNode *match = getXpathResult(xpathObj, lpc);
 168 
 169         CRM_LOG_ASSERT(match != NULL);
 170         if(match == NULL) { continue; };
 171 
 172         op_id = pcmk__xe_id(match);
 173 
 174         if (rsc_op_xpath == NULL) {
 175             rsc_op_xpath = g_string_new(RSC_OP_PREFIX);
 176         } else {
 177             g_string_truncate(rsc_op_xpath, sizeof(RSC_OP_PREFIX) - 1);
 178         }
 179         pcmk__g_strcat(rsc_op_xpath, op_id, "']", NULL);
 180 
 181         op_match = xpath_search(diff, (const char *) rsc_op_xpath->str);
 182         if (numXpathResults(op_match) == 0) {
 183             /* Prevent false positives by matching cancelations too */
 184             const char *node = get_node_id(match);
 185             pcmk__graph_action_t *cancelled = get_cancel_action(op_id, node);
 186 
 187             if (cancelled == NULL) {
 188                 crm_debug("No match for deleted action %s (%s on %s)",
 189                           (const char *) rsc_op_xpath->str, op_id, node);
 190                 abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 191                                  "Resource op removal", match);
 192                 freeXpathObject(op_match);
 193                 goto bail;
 194 
 195             } else {
 196                 crm_debug("Deleted " PCMK__XE_LRM_RSC_OP " %s on %s was for "
 197                           "graph event %d",
 198                           op_id, node, cancelled->id);
 199             }
 200         }
 201 
 202         freeXpathObject(op_match);
 203     }
 204 
 205   bail:
 206     freeXpathObject(xpathObj);
 207     if (rsc_op_xpath != NULL) {
 208         g_string_free(rsc_op_xpath, TRUE);
 209     }
 210 }
 211 
 212 static void
 213 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
 214 {
 215     for (xmlNode *rsc_op = pcmk__xe_first_child(lrm_resource, NULL, NULL, NULL);
 216          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op)) {
 217         process_graph_event(rsc_op, node);
 218     }
 219     if (shutdown_lock_cleared(lrm_resource)) {
 220         // @TODO would be more efficient to abort once after transition done
 221         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 222                          "Shutdown lock cleared", lrm_resource);
 223     }
 224 }
 225 
 226 static void
 227 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
 228                          const char *op, const char *xpath)
 229 {
 230     xmlNode *rsc = NULL;
 231 
 232     if (xml == NULL) {
 233         return;
 234     }
 235 
 236     if (pcmk__xe_is(xml, PCMK__XE_LRM)) {
 237         xml = pcmk__xe_first_child(xml, PCMK__XE_LRM_RESOURCES, NULL, NULL);
 238         CRM_CHECK(xml != NULL, return);
 239     }
 240 
 241     CRM_CHECK(pcmk__xe_is(xml, PCMK__XE_LRM_RESOURCES), return);
 242 
 243     /*
 244      * Updates by, or in response to, TE actions will never contain updates
 245      * for more than one resource at a time, so such updates indicate an
 246      * LRM refresh.
 247      *
 248      * In that case, start a new transition rather than check each result
 249      * individually, which can result in _huge_ speedups in large clusters.
 250      *
 251      * Unfortunately, we can only do so when there are no pending actions.
 252      * Otherwise, we could mistakenly throw away those results here, and
 253      * the cluster will stall waiting for them and time out the operation.
 254      */
 255     if ((controld_globals.transition_graph->pending == 0)
 256         && (xml->children != NULL) && (xml->children->next != NULL)) {
 257 
 258         crm_log_xml_trace(change, "lrm-refresh");
 259         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 260                          "History refresh", NULL);
 261         return;
 262     }
 263 
 264     for (rsc = pcmk__xe_first_child(xml, NULL, NULL, NULL); rsc != NULL;
 265          rsc = pcmk__xe_next(rsc)) {
 266         crm_trace("Processing %s", pcmk__xe_id(rsc));
 267         process_lrm_resource_diff(rsc, node);
 268     }
 269 }
 270 
 271 static char *extract_node_uuid(const char *xpath) 
     /*  */
 272 {
 273     char *mutable_path = pcmk__str_copy(xpath);
 274     char *node_uuid = NULL;
 275     char *search = NULL;
 276     char *match = NULL;
 277 
 278     match = strstr(mutable_path, PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");
 279     if (match == NULL) {
 280         free(mutable_path);
 281         return NULL;
 282     }
 283     match += strlen(PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");
 284 
 285     search = strchr(match, '\'');
 286     if (search == NULL) {
 287         free(mutable_path);
 288         return NULL;
 289     }
 290     search[0] = 0;
 291 
 292     node_uuid = pcmk__str_copy(match);
 293     free(mutable_path);
 294     return node_uuid;
 295 }
 296 
 297 static void
 298 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 299                   const char *reason)
 300 {
 301     char *node_uuid = NULL;
 302     pcmk__graph_action_t *down = NULL;
 303 
 304     if (!pcmk__str_eq(op, PCMK_VALUE_DELETE, pcmk__str_none)) {
 305         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 306                          change);
 307         return;
 308     }
 309 
 310     node_uuid = extract_node_uuid(xpath);
 311     if(node_uuid == NULL) {
 312         crm_err("Could not extract node ID from %s", xpath);
 313         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 314                          change);
 315         return;
 316     }
 317 
 318     down = match_down_event(node_uuid);
 319     if (down == NULL) {
 320         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 321         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 322                          change);
 323     } else {
 324         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 325     }
 326     free(node_uuid);
 327 }
 328 
 329 static void
 330 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 331 {
 332     char *mutable_key = pcmk__str_copy(xpath);
 333     char *key;
 334     char *node_uuid;
 335 
 336     // Extract the part of xpath between last pair of single quotes
 337     key = strrchr(mutable_key, '\'');
 338     if (key != NULL) {
 339         *key = '\0';
 340         key = strrchr(mutable_key, '\'');
 341     }
 342     if (key == NULL) {
 343         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 344                  xpath);
 345         free(mutable_key);
 346         return;
 347     }
 348     ++key;
 349 
 350     node_uuid = extract_node_uuid(xpath);
 351     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 352         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 353                          "Resource operation removal", change);
 354     }
 355     free(mutable_key);
 356     free(node_uuid);
 357 }
 358 
 359 static void
 360 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 361 {
 362     if (strstr(xpath, "/" PCMK__XE_LRM_RSC_OP "[")) {
 363         process_op_deletion(xpath, change);
 364 
 365     } else if (strstr(xpath, "/" PCMK__XE_LRM "[")) {
 366         abort_unless_down(xpath, op, change, "Resource state removal");
 367 
 368     } else if (strstr(xpath, "/" PCMK__XE_NODE_STATE "[")) {
 369         abort_unless_down(xpath, op, change, "Node state removal");
 370 
 371     } else {
 372         crm_trace("Ignoring delete of %s", xpath);
 373     }
 374 }
 375 
 376 static void
 377 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 378                         const char *xpath)
 379 {
 380     xmlNode *lrm = pcmk__xe_first_child(state, PCMK__XE_LRM, NULL, NULL);
 381 
 382     process_resource_updates(pcmk__xe_id(state), lrm, change, op, xpath);
 383 }
 384 
 385 static void
 386 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 387                     const char *xpath)
 388 {
 389     for (xmlNode *state = pcmk__xe_first_child(status, NULL, NULL, NULL);
 390          state != NULL; state = pcmk__xe_next(state)) {
 391 
 392         process_node_state_diff(state, change, op, xpath);
 393     }
 394 }
 395 
 396 static void
 397 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 398                  const char *xpath)
 399 {
 400     xmlNode *status = pcmk__xe_first_child(cib, PCMK_XE_STATUS, NULL, NULL);
 401     xmlNode *config = pcmk__xe_first_child(cib, PCMK_XE_CONFIGURATION, NULL,
 402                                            NULL);
 403 
 404     if (status) {
 405         process_status_diff(status, change, op, xpath);
 406     }
 407     if (config) {
 408         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 409                          "Non-status-only change", change);
 410     }
 411 }
 412 
 413 static int
 414 te_update_diff_element_v2(xmlNode *change, void *userdata)
     /*  */
 415 {
 416     xmlNode *match = NULL;
 417     const char *name = NULL;
 418     const char *xpath = crm_element_value(change, PCMK_XA_PATH);
 419 
 420     // Possible ops: create, modify, delete, move
 421     const char *op = crm_element_value(change, PCMK_XA_OPERATION);
 422 
 423     // Ignore uninteresting updates
 424     if (op == NULL) {
 425         return pcmk_rc_ok;
 426 
 427     } else if (xpath == NULL) {
 428         crm_trace("Ignoring %s change for version field", op);
 429         return pcmk_rc_ok;
 430 
 431     } else if ((strcmp(op, PCMK_VALUE_MOVE) == 0)
 432                && (strstr(xpath,
 433                           "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
 434                           "/" PCMK_XE_RESOURCES) == NULL)) {
 435         /* We still need to consider moves within the resources section,
 436          * since they affect placement order.
 437          */
 438         crm_trace("Ignoring move change at %s", xpath);
 439         return pcmk_rc_ok;
 440     }
 441 
 442     // Find the result of create/modify ops
 443     if (strcmp(op, PCMK_VALUE_CREATE) == 0) {
 444         match = change->children;
 445 
 446     } else if (strcmp(op, PCMK_VALUE_MODIFY) == 0) {
 447         match = pcmk__xe_first_child(change, PCMK_XE_CHANGE_RESULT, NULL, NULL);
 448         if(match) {
 449             match = match->children;
 450         }
 451 
 452     } else if (!pcmk__str_any_of(op,
 453                                  PCMK_VALUE_DELETE, PCMK_VALUE_MOVE,
 454                                  NULL)) {
 455         crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 456                  op, xpath);
 457         return pcmk_rc_ok;
 458     }
 459 
 460     if (match) {
 461         if (match->type == XML_COMMENT_NODE) {
 462             crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 463             return pcmk_rc_ok;
 464         }
 465         name = (const char *)match->name;
 466     }
 467 
 468     crm_trace("Handling %s operation for %s%s%s",
 469               op, (xpath? xpath : "CIB"),
 470               (name? " matched by " : ""), (name? name : ""));
 471 
 472     if (strstr(xpath, "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION)) {
 473         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 474                          "Configuration change", change);
 475         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 476 
 477     } else if (strstr(xpath, "/" PCMK_XE_TICKETS)
 478                || pcmk__str_eq(name, PCMK_XE_TICKETS, pcmk__str_none)) {
 479         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 480                          "Ticket attribute change", change);
 481         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 482 
 483     } else if (strstr(xpath, "/" PCMK__XE_TRANSIENT_ATTRIBUTES "[")
 484                || pcmk__str_eq(name, PCMK__XE_TRANSIENT_ATTRIBUTES,
 485                                pcmk__str_none)) {
 486         abort_unless_down(xpath, op, change, "Transient attribute change");
 487         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 488 
 489     } else if (strcmp(op, PCMK_VALUE_DELETE) == 0) {
 490         process_delete_diff(xpath, op, change);
 491 
 492     } else if (name == NULL) {
 493         crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 494                  op, xpath);
 495 
 496     } else if (strcmp(name, PCMK_XE_CIB) == 0) {
 497         process_cib_diff(match, change, op, xpath);
 498 
 499     } else if (strcmp(name, PCMK_XE_STATUS) == 0) {
 500         process_status_diff(match, change, op, xpath);
 501 
 502     } else if (strcmp(name, PCMK__XE_NODE_STATE) == 0) {
 503         process_node_state_diff(match, change, op, xpath);
 504 
 505     } else if (strcmp(name, PCMK__XE_LRM) == 0) {
 506         process_resource_updates(pcmk__xe_id(match), match, change, op,
 507                                  xpath);
 508 
 509     } else if (strcmp(name, PCMK__XE_LRM_RESOURCES) == 0) {
 510         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 511 
 512         process_resource_updates(local_node, match, change, op, xpath);
 513         free(local_node);
 514 
 515     } else if (strcmp(name, PCMK__XE_LRM_RESOURCE) == 0) {
 516         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 517 
 518         process_lrm_resource_diff(match, local_node);
 519         free(local_node);
 520 
 521     } else if (strcmp(name, PCMK__XE_LRM_RSC_OP) == 0) {
 522         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 523 
 524         process_graph_event(match, local_node);
 525         free(local_node);
 526 
 527     } else {
 528         crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 529                  op, xpath, name);
 530     }
 531 
 532     return pcmk_rc_ok;
 533 }
 534 
 535 static void
 536 te_update_diff_v2(xmlNode *diff)
     /*  */
 537 {
 538     crm_log_xml_trace(diff, "Patch:Raw");
 539     pcmk__xe_foreach_child(diff, NULL, te_update_diff_element_v2, NULL);
 540 }
 541 
 542 void
 543 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 544 {
 545     xmlNode *wrapper = NULL;
 546     xmlNode *diff = NULL;
 547     const char *op = NULL;
 548     int rc = -EINVAL;
 549     int format = 1;
 550     int p_add[] = { 0, 0, 0 };
 551     int p_del[] = { 0, 0, 0 };
 552 
 553     CRM_CHECK(msg != NULL, return);
 554     crm_element_value_int(msg, PCMK__XA_CIB_RC, &rc);
 555 
 556     if (controld_globals.transition_graph == NULL) {
 557         crm_trace("No graph");
 558         return;
 559 
 560     } else if (rc < pcmk_ok) {
 561         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 562         return;
 563 
 564     } else if (controld_globals.transition_graph->complete
 565                && (controld_globals.fsa_state != S_IDLE)
 566                && (controld_globals.fsa_state != S_TRANSITION_ENGINE)
 567                && (controld_globals.fsa_state != S_POLICY_ENGINE)) {
 568         crm_trace("Filter state=%s (complete)",
 569                   fsa_state2string(controld_globals.fsa_state));
 570         return;
 571     }
 572 
 573     op = crm_element_value(msg, PCMK__XA_CIB_OP);
 574 
 575     wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT, NULL, NULL);
 576     diff = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 577 
 578     xml_patch_versions(diff, p_add, p_del);
 579     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 580               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 581               fsa_state2string(controld_globals.fsa_state));
 582 
 583     crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
 584     switch (format) {
 585         case 1:
 586             te_update_diff_v1(event, diff);
 587             break;
 588         case 2:
 589             te_update_diff_v2(diff);
 590             break;
 591         default:
 592             crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 593                      format);
 594     }
 595     controld_remove_all_outside_events();
 596 }
 597 
 598 void
 599 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 600 {
 601     const char *value = NULL;
 602     xmlXPathObject *xpathObj = NULL;
 603     int nmatches = 0;
 604 
 605     CRM_CHECK(msg != NULL, return);
 606 
 607     // Transition requests must specify transition engine as subsystem
 608     value = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
 609     if (pcmk__str_empty(value)
 610         || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
 611         crm_info("Received invalid transition request: subsystem '%s' not '"
 612                  CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
 613         return;
 614     }
 615 
 616     // Only the lrm_invoke command is supported as a transition request
 617     value = crm_element_value(msg, PCMK__XA_CRM_TASK);
 618     if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
 619         crm_info("Received invalid transition request: command '%s' not '"
 620                  CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
 621         return;
 622     }
 623 
 624     // Transition requests must be marked as coming from the executor
 625     value = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
 626     if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
 627         crm_info("Received invalid transition request: from '%s' not '"
 628                  CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
 629         return;
 630     }
 631 
 632     crm_debug("Processing transition request with ref='%s' origin='%s'",
 633               pcmk__s(crm_element_value(msg, PCMK_XA_REFERENCE), ""),
 634               pcmk__s(crm_element_value(msg, PCMK__XA_SRC), ""));
 635 
 636     xpathObj = xpath_search(xml_data, "//" PCMK__XE_LRM_RSC_OP);
 637     nmatches = numXpathResults(xpathObj);
 638     if (nmatches == 0) {
 639         crm_err("Received transition request with no results (bug?)");
 640     } else {
 641         for (int lpc = 0; lpc < nmatches; lpc++) {
 642             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 643             const char *node = get_node_id(rsc_op);
 644 
 645             process_graph_event(rsc_op, node);
 646         }
 647     }
 648     freeXpathObject(xpathObj);
 649 }
 650 
 651 void
 652 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 653 {
 654     if (rc < pcmk_ok) {
 655         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 656     }
 657 }
 658 
 659 /*!
 660  * \brief Handle a timeout in node-to-node communication
 661  *
 662  * \param[in,out] data  Pointer to graph action
 663  *
 664  * \return FALSE (indicating that source should be not be re-added)
 665  */
 666 gboolean
 667 action_timer_callback(gpointer data)
     /*  */
 668 {
 669     pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
 670     const char *task = NULL;
 671     const char *on_node = NULL;
 672     const char *via_node = NULL;
 673 
 674     CRM_CHECK(data != NULL, return FALSE);
 675 
 676     stop_te_timer(action);
 677 
 678     task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 679     on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
 680     via_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 681 
 682     if (controld_globals.transition_graph->complete) {
 683         crm_notice("Node %s did not send %s result (via %s) within %dms "
 684                    "(ignoring because transition not in progress)",
 685                    (on_node? on_node : ""), (task? task : "unknown action"),
 686                    (via_node? via_node : "controller"), action->timeout);
 687     } else {
 688         /* fail the action */
 689 
 690         crm_err("Node %s did not send %s result (via %s) within %dms "
 691                 "(action timeout plus " PCMK_OPT_CLUSTER_DELAY ")",
 692                 (on_node? on_node : ""), (task? task : "unknown action"),
 693                 (via_node? via_node : "controller"),
 694                 (action->timeout
 695                  + controld_globals.transition_graph->network_delay));
 696         pcmk__log_graph_action(LOG_ERR, action);
 697 
 698         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
 699 
 700         te_action_confirmed(action, controld_globals.transition_graph);
 701         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 702                          "Action lost", NULL);
 703 
 704         // Record timeout in the CIB if appropriate
 705         if ((action->type == pcmk__rsc_graph_action)
 706             && controld_action_is_recordable(task)) {
 707             controld_record_action_timeout(action);
 708         }
 709     }
 710 
 711     return FALSE;
 712 }

/* */