root/daemons/controld/controld_te_callbacks.c

/* */

DEFINITIONS

1. shutdown_lock_cleared
2. te_update_diff_v1
3. process_lrm_resource_diff
4. process_resource_updates
5. get_node_from_xpath
6. extract_node_uuid
7. abort_unless_down
8. process_op_deletion
9. process_delete_diff
10. process_node_state_diff
11. process_status_diff
12. process_cib_diff
13. te_update_diff_v2
14. te_update_diff
15. process_te_message
16. cib_action_updated
17. action_timer_callback

   1 /*
   2  * Copyright 2004-2020 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/cluster.h>        /* For ONLINESTATUS etc */
  19 
  20 #include <pacemaker-controld.h>
  21 
  22 void te_update_confirm(const char *event, xmlNode * msg);
  23 
  24 extern char *te_uuid;
  25 gboolean shuttingdown = FALSE;
  26 crm_graph_t *transition_graph;
  27 crm_trigger_t *transition_trigger = NULL;
  28 
  29 /* #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */
  30 #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"
  31 
  32 // An explicit shutdown-lock of 0 means the lock has been cleared
  33 static bool
  34 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  35 {
  36     time_t shutdown_lock = 0;
  37 
  38     return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
  39                                     &shutdown_lock) == pcmk_ok)
  40            && (shutdown_lock == 0);
  41 }
  42 
  43 static void
  44 te_update_diff_v1(const char *event, xmlNode *diff)
     /*  */
  45 {
  46     int lpc, max;
  47     xmlXPathObject *xpathObj = NULL;
  48 
  49     CRM_CHECK(diff != NULL, return);
  50 
  51     xml_log_patchset(LOG_TRACE, __func__, diff);
  52     if (cib_config_changed(NULL, NULL, &diff)) {
  53         abort_transition(INFINITY, tg_restart, "Non-status change", diff);
  54         goto bail;              /* configuration changed */
  55     }
  56 
  57     /* Tickets Attributes - Added/Updated */
  58     xpathObj =
  59         xpath_search(diff,
  60                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
  61     if (numXpathResults(xpathObj) > 0) {
  62         xmlNode *aborted = getXpathResult(xpathObj, 0);
  63 
  64         abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
  65         goto bail;
  66 
  67     }
  68     freeXpathObject(xpathObj);
  69 
  70     /* Tickets Attributes - Removed */
  71     xpathObj =
  72         xpath_search(diff,
  73                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
  74     if (numXpathResults(xpathObj) > 0) {
  75         xmlNode *aborted = getXpathResult(xpathObj, 0);
  76 
  77         abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
  78         goto bail;
  79     }
  80     freeXpathObject(xpathObj);
  81 
  82     /* Transient Attributes - Added/Updated */
  83     xpathObj =
  84         xpath_search(diff,
  85                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
  86                      XML_TAG_TRANSIENT_NODEATTRS "//" XML_CIB_TAG_NVPAIR);
  87     max = numXpathResults(xpathObj);
  88 
  89     for (lpc = 0; lpc < max; lpc++) {
  90         xmlNode *attr = getXpathResult(xpathObj, lpc);
  91         const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
  92         const char *value = NULL;
  93 
  94         if (pcmk__str_eq(CRM_OP_PROBED, name, pcmk__str_casei)) {
  95             value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
  96         }
  97 
  98         if (crm_is_true(value) == FALSE) {
  99             abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
 100             crm_log_xml_trace(attr, "Abort");
 101             goto bail;
 102         }
 103     }
 104 
 105     freeXpathObject(xpathObj);
 106 
 107     /* Transient Attributes - Removed */
 108     xpathObj =
 109         xpath_search(diff,
 110                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
 111                      XML_TAG_TRANSIENT_NODEATTRS);
 112     if (numXpathResults(xpathObj) > 0) {
 113         xmlNode *aborted = getXpathResult(xpathObj, 0);
 114 
 115         abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
 116         goto bail;
 117 
 118     }
 119     freeXpathObject(xpathObj);
 120 
 121     // Check for lrm_resource entries
 122     xpathObj = xpath_search(diff,
 123                             "//" F_CIB_UPDATE_RESULT
 124                             "//" XML_TAG_DIFF_ADDED
 125                             "//" XML_LRM_TAG_RESOURCE);
 126     max = numXpathResults(xpathObj);
 127 
 128     /*
 129      * Updates by, or in response to, graph actions will never affect more than
 130      * one resource at a time, so such updates indicate an LRM refresh. In that
 131      * case, start a new transition rather than check each result individually,
 132      * which can result in _huge_ speedups in large clusters.
 133      *
 134      * Unfortunately, we can only do so when there are no pending actions.
 135      * Otherwise, we could mistakenly throw away those results here, and
 136      * the cluster will stall waiting for them and time out the operation.
 137      */
 138     if ((transition_graph->pending == 0) && (max > 1)) {
 139         crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
 140                   max);
 141         crm_log_xml_trace(diff, "lrm-refresh");
 142         abort_transition(INFINITY, tg_restart, "History refresh", NULL);
 143         goto bail;
 144     }
 145 
 146     if (max == 1) {
 147         xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
 148 
 149         if (shutdown_lock_cleared(lrm_resource)) {
 150             // @TODO would be more efficient to abort once after transition done
 151             abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
 152                              lrm_resource);
 153             // Still process results, so we stop timers and update failcounts
 154         }
 155     }
 156     freeXpathObject(xpathObj);
 157 
 158     /* Process operation updates */
 159     xpathObj =
 160         xpath_search(diff,
 161                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
 162     max = numXpathResults(xpathObj);
 163     if (max > 0) {
 164         int lpc = 0;
 165 
 166         for (lpc = 0; lpc < max; lpc++) {
 167             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 168             const char *node = get_node_id(rsc_op);
 169 
 170             process_graph_event(rsc_op, node);
 171         }
 172     }
 173     freeXpathObject(xpathObj);
 174 
 175     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
 176     xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
 177     max = numXpathResults(xpathObj);
 178     for (lpc = 0; lpc < max; lpc++) {
 179         int path_max = 0;
 180         const char *op_id = NULL;
 181         char *rsc_op_xpath = NULL;
 182         xmlXPathObject *op_match = NULL;
 183         xmlNode *match = getXpathResult(xpathObj, lpc);
 184 
 185         CRM_LOG_ASSERT(match != NULL);
 186         if(match == NULL) { continue; };
 187 
 188         op_id = ID(match);
 189 
 190         path_max = strlen(RSC_OP_TEMPLATE) + strlen(op_id) + 1;
 191         rsc_op_xpath = calloc(1, path_max);
 192         snprintf(rsc_op_xpath, path_max, RSC_OP_TEMPLATE, op_id);
 193 
 194         op_match = xpath_search(diff, rsc_op_xpath);
 195         if (numXpathResults(op_match) == 0) {
 196             /* Prevent false positives by matching cancelations too */
 197             const char *node = get_node_id(match);
 198             crm_action_t *cancelled = get_cancel_action(op_id, node);
 199 
 200             if (cancelled == NULL) {
 201                 crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
 202                           node);
 203                 abort_transition(INFINITY, tg_restart, "Resource op removal", match);
 204                 freeXpathObject(op_match);
 205                 free(rsc_op_xpath);
 206                 goto bail;
 207 
 208             } else {
 209                 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
 210                           op_id, node, cancelled->id);
 211             }
 212         }
 213 
 214         freeXpathObject(op_match);
 215         free(rsc_op_xpath);
 216     }
 217 
 218   bail:
 219     freeXpathObject(xpathObj);
 220 }
 221 
 222 static void
 223 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
 224 {
 225     for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
 226          rsc_op = pcmk__xml_next(rsc_op)) {
 227         process_graph_event(rsc_op, node);
 228     }
 229     if (shutdown_lock_cleared(lrm_resource)) {
 230         // @TODO would be more efficient to abort once after transition done
 231         abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
 232                          lrm_resource);
 233     }
 234 }
 235 
 236 static void
 237 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
 238                          const char *op, const char *xpath)
 239 {
 240     xmlNode *rsc = NULL;
 241 
 242     if (xml == NULL) {
 243         return;
 244 
 245     } else if (strcmp((const char*)xml->name, XML_CIB_TAG_LRM) == 0) {
 246         xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
 247         crm_trace("Got %p in %s", xml, XML_CIB_TAG_LRM);
 248     }
 249 
 250     CRM_ASSERT(strcmp((const char*)xml->name, XML_LRM_TAG_RESOURCES) == 0);
 251 
 252     /*
 253      * Updates by, or in response to, TE actions will never contain updates
 254      * for more than one resource at a time, so such updates indicate an
 255      * LRM refresh.
 256      *
 257      * In that case, start a new transition rather than check each result
 258      * individually, which can result in _huge_ speedups in large clusters.
 259      *
 260      * Unfortunately, we can only do so when there are no pending actions.
 261      * Otherwise, we could mistakenly throw away those results here, and
 262      * the cluster will stall waiting for them and time out the operation.
 263      */
 264     if ((transition_graph->pending == 0)
 265         && xml->children && xml->children->next) {
 266 
 267         crm_log_xml_trace(change, "lrm-refresh");
 268         abort_transition(INFINITY, tg_restart, "History refresh", NULL);
 269         return;
 270     }
 271 
 272     for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
 273          rsc = pcmk__xml_next(rsc)) {
 274         crm_trace("Processing %s", ID(rsc));
 275         process_lrm_resource_diff(rsc, node);
 276     }
 277 }
 278 
 279 #define NODE_PATT "/lrm[@id="
 280 static char *get_node_from_xpath(const char *xpath) 
     /*  */
 281 {
 282     char *nodeid = NULL;
 283     char *tmp = strstr(xpath, NODE_PATT);
 284 
 285     if(tmp) {
 286         tmp += strlen(NODE_PATT);
 287         tmp += 1;
 288 
 289         nodeid = strdup(tmp);
 290         tmp = strstr(nodeid, "\'");
 291         CRM_ASSERT(tmp);
 292         tmp[0] = 0;
 293     }
 294     return nodeid;
 295 }
 296 
 297 static char *extract_node_uuid(const char *xpath) 
     /*  */
 298 {
 299     char *mutable_path = strdup(xpath);
 300     char *node_uuid = NULL;
 301     char *search = NULL;
 302     char *match = NULL;
 303 
 304     match = strstr(mutable_path, "node_state[@id=\'");
 305     if (match == NULL) {
 306         free(mutable_path);
 307         return NULL;
 308     }
 309     match += strlen("node_state[@id=\'");
 310 
 311     search = strchr(match, '\'');
 312     if (search == NULL) {
 313         free(mutable_path);
 314         return NULL;
 315     }
 316     search[0] = 0;
 317 
 318     node_uuid = strdup(match);
 319     free(mutable_path);
 320     return node_uuid;
 321 }
 322 
 323 static void
 324 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 325                   const char *reason)
 326 {
 327     char *node_uuid = NULL;
 328     crm_action_t *down = NULL;
 329 
 330     if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
 331         abort_transition(INFINITY, tg_restart, reason, change);
 332         return;
 333     }
 334 
 335     node_uuid = extract_node_uuid(xpath);
 336     if(node_uuid == NULL) {
 337         crm_err("Could not extract node ID from %s", xpath);
 338         abort_transition(INFINITY, tg_restart, reason, change);
 339         return;
 340     }
 341 
 342     down = match_down_event(node_uuid);
 343     if (down == NULL) {
 344         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 345         abort_transition(INFINITY, tg_restart, reason, change);
 346     } else {
 347         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 348     }
 349     free(node_uuid);
 350 }
 351 
 352 static void
 353 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 354 {
 355     char *mutable_key = strdup(xpath);
 356     char *key;
 357     char *node_uuid;
 358 
 359     // Extract the part of xpath between last pair of single quotes
 360     key = strrchr(mutable_key, '\'');
 361     if (key != NULL) {
 362         *key = '\0';
 363         key = strrchr(mutable_key, '\'');
 364     }
 365     if (key == NULL) {
 366         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 367                  xpath);
 368         free(mutable_key);
 369         return;
 370     }
 371     ++key;
 372 
 373     node_uuid = extract_node_uuid(xpath);
 374     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 375         abort_transition(INFINITY, tg_restart, "Resource operation removal",
 376                          change);
 377     }
 378     free(mutable_key);
 379     free(node_uuid);
 380 }
 381 
 382 static void
 383 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 384 {
 385     if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
 386         process_op_deletion(xpath, change);
 387 
 388     } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
 389         abort_unless_down(xpath, op, change, "Resource state removal");
 390 
 391     } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
 392         abort_unless_down(xpath, op, change, "Node state removal");
 393 
 394     } else {
 395         crm_trace("Ignoring delete of %s", xpath);
 396     }
 397 }
 398 
 399 static void
 400 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 401                         const char *xpath)
 402 {
 403     xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
 404 
 405     process_resource_updates(ID(state), lrm, change, op, xpath);
 406 }
 407 
 408 static void
 409 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 410                     const char *xpath)
 411 {
 412     for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
 413          state = pcmk__xml_next(state)) {
 414         process_node_state_diff(state, change, op, xpath);
 415     }
 416 }
 417 
 418 static void
 419 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 420                  const char *xpath)
 421 {
 422     xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
 423     xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
 424 
 425     if (status) {
 426         process_status_diff(status, change, op, xpath);
 427     }
 428     if (config) {
 429         abort_transition(INFINITY, tg_restart,
 430                          "Non-status-only change", change);
 431     }
 432 }
 433 
 434 static void
 435 te_update_diff_v2(xmlNode *diff)
     /*  */
 436 {
 437     crm_log_xml_trace(diff, "Patch:Raw");
 438 
 439     for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
 440          change = pcmk__xml_next(change)) {
 441 
 442         xmlNode *match = NULL;
 443         const char *name = NULL;
 444         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 445 
 446         // Possible ops: create, modify, delete, move
 447         const char *op = crm_element_value(change, XML_DIFF_OP);
 448 
 449         // Ignore uninteresting updates
 450         if (op == NULL) {
 451             continue;
 452 
 453         } else if (xpath == NULL) {
 454             crm_trace("Ignoring %s change for version field", op);
 455             continue;
 456 
 457         } else if (strcmp(op, "move") == 0) {
 458             crm_trace("Ignoring move change at %s", xpath);
 459             continue;
 460         }
 461 
 462         // Find the result of create/modify ops
 463         if (strcmp(op, "create") == 0) {
 464             match = change->children;
 465 
 466         } else if (strcmp(op, "modify") == 0) {
 467             match = first_named_child(change, XML_DIFF_RESULT);
 468             if(match) {
 469                 match = match->children;
 470             }
 471 
 472         } else if (strcmp(op, "delete") != 0) {
 473             crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 474                      op, xpath);
 475             continue;
 476         }
 477 
 478         if (match) {
 479             if (match->type == XML_COMMENT_NODE) {
 480                 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 481                 continue;
 482             }
 483             name = (const char *)match->name;
 484         }
 485 
 486         crm_trace("Handling %s operation for %s%s%s",
 487                   op, (xpath? xpath : "CIB"),
 488                   (name? " matched by " : ""), (name? name : ""));
 489 
 490         if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
 491             abort_transition(INFINITY, tg_restart, "Configuration change",
 492                              change);
 493             break; // Won't be packaged with operation results we may be waiting for
 494 
 495         } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
 496                    || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
 497             abort_transition(INFINITY, tg_restart, "Ticket attribute change", change);
 498             break; // Won't be packaged with operation results we may be waiting for
 499 
 500         } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
 501                    || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS, pcmk__str_casei)) {
 502             abort_unless_down(xpath, op, change, "Transient attribute change");
 503             break; // Won't be packaged with operation results we may be waiting for
 504 
 505         } else if (strcmp(op, "delete") == 0) {
 506             process_delete_diff(xpath, op, change);
 507 
 508         } else if (name == NULL) {
 509             crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 510                      op, xpath);
 511 
 512         } else if (strcmp(name, XML_TAG_CIB) == 0) {
 513             process_cib_diff(match, change, op, xpath);
 514 
 515         } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
 516             process_status_diff(match, change, op, xpath);
 517 
 518         } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
 519             process_node_state_diff(match, change, op, xpath);
 520 
 521         } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
 522             process_resource_updates(ID(match), match, change, op, xpath);
 523 
 524         } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
 525             char *local_node = get_node_from_xpath(xpath);
 526 
 527             process_resource_updates(local_node, match, change, op, xpath);
 528             free(local_node);
 529 
 530         } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
 531             char *local_node = get_node_from_xpath(xpath);
 532 
 533             process_lrm_resource_diff(match, local_node);
 534             free(local_node);
 535 
 536         } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
 537             char *local_node = get_node_from_xpath(xpath);
 538 
 539             process_graph_event(match, local_node);
 540             free(local_node);
 541 
 542         } else {
 543             crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 544                      op, xpath, name);
 545         }
 546     }
 547 }
 548 
 549 void
 550 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 551 {
 552     xmlNode *diff = NULL;
 553     const char *op = NULL;
 554     int rc = -EINVAL;
 555     int format = 1;
 556     int p_add[] = { 0, 0, 0 };
 557     int p_del[] = { 0, 0, 0 };
 558 
 559     CRM_CHECK(msg != NULL, return);
 560     crm_element_value_int(msg, F_CIB_RC, &rc);
 561 
 562     if (transition_graph == NULL) {
 563         crm_trace("No graph");
 564         return;
 565 
 566     } else if (rc < pcmk_ok) {
 567         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 568         return;
 569 
 570     } else if (transition_graph->complete
 571                && fsa_state != S_IDLE
 572                && fsa_state != S_TRANSITION_ENGINE
 573                && fsa_state != S_POLICY_ENGINE) {
 574         crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
 575                   transition_graph->complete);
 576         return;
 577     }
 578 
 579     op = crm_element_value(msg, F_CIB_OPERATION);
 580     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 581 
 582     xml_patch_versions(diff, p_add, p_del);
 583     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 584               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 585               fsa_state2string(fsa_state));
 586 
 587     crm_element_value_int(diff, "format", &format);
 588     switch (format) {
 589         case 1:
 590             te_update_diff_v1(event, diff);
 591             break;
 592         case 2:
 593             te_update_diff_v2(diff);
 594             break;
 595         default:
 596             crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 597                      format);
 598     }
 599 }
 600 
 601 gboolean
 602 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 603 {
 604     const char *from = crm_element_value(msg, F_ORIG);
 605     const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 606     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 607     const char *ref = crm_element_value(msg, F_CRM_REFERENCE);
 608     const char *op = crm_element_value(msg, F_CRM_TASK);
 609     const char *type = crm_element_value(msg, F_CRM_MSG_TYPE);
 610 
 611     crm_trace("Processing %s (%s) message", op, ref);
 612     crm_log_xml_trace(msg, "ipc");
 613 
 614     if (op == NULL) {
 615         /* error */
 616 
 617     } else if (sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) {
 618         crm_trace("Bad sys-to %s", crm_str(sys_to));
 619         return FALSE;
 620 
 621     } else if (pcmk__str_eq(op, CRM_OP_INVOKE_LRM, pcmk__str_casei)
 622                && pcmk__str_eq(sys_from, CRM_SYSTEM_LRMD, pcmk__str_casei)
 623 /*                && pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_casei) */
 624         ) {
 625         xmlXPathObject *xpathObj = NULL;
 626 
 627         crm_log_xml_trace(msg, "Processing (N)ACK");
 628         crm_debug("Processing (N)ACK %s from %s", crm_element_value(msg, F_CRM_REFERENCE), from);
 629 
 630         xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
 631         if (numXpathResults(xpathObj)) {
 632             int lpc = 0, max = numXpathResults(xpathObj);
 633 
 634             for (lpc = 0; lpc < max; lpc++) {
 635                 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 636                 const char *node = get_node_id(rsc_op);
 637 
 638                 process_graph_event(rsc_op, node);
 639             }
 640             freeXpathObject(xpathObj);
 641 
 642         } else {
 643             crm_log_xml_err(msg, "Invalid (N)ACK");
 644             freeXpathObject(xpathObj);
 645             return FALSE;
 646         }
 647 
 648     } else {
 649         crm_err("Unknown command: %s::%s from %s", type, op, sys_from);
 650     }
 651 
 652     crm_trace("finished processing message");
 653 
 654     return TRUE;
 655 }
 656 
 657 void
 658 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 659 {
 660     if (rc < pcmk_ok) {
 661         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 662     }
 663 }
 664 
 665 /*!
 666  * \brief Handle a timeout in node-to-node communication
 667  *
 668  * \param[in] data  Pointer to action timer
 669  *
 670  * \return FALSE (indicating that source should be not be re-added)
 671  */
 672 gboolean
 673 action_timer_callback(gpointer data)
     /*  */
 674 {
 675     crm_action_timer_t *timer = NULL;
 676     const char *task = NULL;
 677     const char *on_node = NULL;
 678     const char *via_node = NULL;
 679 
 680     CRM_CHECK(data != NULL, return FALSE);
 681 
 682     timer = (crm_action_timer_t *) data;
 683     stop_te_timer(timer);
 684 
 685     CRM_CHECK(timer->action != NULL, return FALSE);
 686 
 687     task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
 688     on_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_TARGET);
 689     via_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_ROUTER_NODE);
 690 
 691     if (transition_graph->complete) {
 692         crm_notice("Node %s did not send %s result (via %s) within %dms "
 693                    "(ignoring because transition not in progress)",
 694                    (on_node? on_node : ""), (task? task : "unknown action"),
 695                    (via_node? via_node : "controller"), timer->timeout);
 696     } else {
 697         /* fail the action */
 698 
 699         crm_err("Node %s did not send %s result (via %s) within %dms "
 700                 "(action timeout plus cluster-delay)",
 701                 (on_node? on_node : ""), (task? task : "unknown action"),
 702                 (via_node? via_node : "controller"),
 703                 timer->timeout + transition_graph->network_delay);
 704         print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action);
 705 
 706         timer->action->failed = TRUE;
 707         te_action_confirmed(timer->action, transition_graph);
 708         abort_transition(INFINITY, tg_restart, "Action lost", NULL);
 709 
 710         // Record timeout in the CIB if appropriate
 711         if ((timer->action->type == action_type_rsc)
 712             && controld_action_is_recordable(task)) {
 713             controld_record_action_timeout(timer->action);
 714         }
 715     }
 716 
 717     return FALSE;
 718 }

/* */