root/daemons/controld/controld_te_callbacks.c

/* */

DEFINITIONS

1. shutdown_lock_cleared
2. te_update_diff_v1
3. process_lrm_resource_diff
4. process_resource_updates
5. extract_node_uuid
6. abort_unless_down
7. process_op_deletion
8. process_delete_diff
9. process_node_state_diff
10. process_status_diff
11. process_cib_diff
12. te_update_diff_v2
13. te_update_diff
14. process_te_message
15. cib_action_updated
16. action_timer_callback

   1 /*
   2  * Copyright 2004-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 #include <crm/msg_xml.h>
  18 #include <crm/cluster.h>        /* For ONLINESTATUS etc */
  19 
  20 #include <pacemaker-controld.h>
  21 
  22 void te_update_confirm(const char *event, xmlNode * msg);
  23 
  24 extern char *te_uuid;
  25 gboolean shuttingdown = FALSE;
  26 crm_graph_t *transition_graph;
  27 crm_trigger_t *transition_trigger = NULL;
  28 
  29 /* #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_CIB_TAG_STATE"[@uname='%s']"//"XML_LRM_TAG_RSC_OP"[@id='%s]" */
  30 #define RSC_OP_TEMPLATE "//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB"//"XML_LRM_TAG_RSC_OP"[@id='%s']"
  31 
  32 // An explicit shutdown-lock of 0 means the lock has been cleared
  33 static bool
  34 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  35 {
  36     time_t shutdown_lock = 0;
  37 
  38     return (crm_element_value_epoch(lrm_resource, XML_CONFIG_ATTR_SHUTDOWN_LOCK,
  39                                     &shutdown_lock) == pcmk_ok)
  40            && (shutdown_lock == 0);
  41 }
  42 
  43 static void
  44 te_update_diff_v1(const char *event, xmlNode *diff)
     /*  */
  45 {
  46     int lpc, max;
  47     xmlXPathObject *xpathObj = NULL;
  48 
  49     CRM_CHECK(diff != NULL, return);
  50 
  51     xml_log_patchset(LOG_TRACE, __func__, diff);
  52     if (cib_config_changed(NULL, NULL, &diff)) {
  53         abort_transition(INFINITY, tg_restart, "Non-status change", diff);
  54         goto bail;              /* configuration changed */
  55     }
  56 
  57     /* Tickets Attributes - Added/Updated */
  58     xpathObj =
  59         xpath_search(diff,
  60                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
  61     if (numXpathResults(xpathObj) > 0) {
  62         xmlNode *aborted = getXpathResult(xpathObj, 0);
  63 
  64         abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
  65         goto bail;
  66 
  67     }
  68     freeXpathObject(xpathObj);
  69 
  70     /* Tickets Attributes - Removed */
  71     xpathObj =
  72         xpath_search(diff,
  73                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
  74     if (numXpathResults(xpathObj) > 0) {
  75         xmlNode *aborted = getXpathResult(xpathObj, 0);
  76 
  77         abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
  78         goto bail;
  79     }
  80     freeXpathObject(xpathObj);
  81 
  82     /* Transient Attributes - Removed */
  83     xpathObj =
  84         xpath_search(diff,
  85                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
  86                      XML_TAG_TRANSIENT_NODEATTRS);
  87     if (numXpathResults(xpathObj) > 0) {
  88         xmlNode *aborted = getXpathResult(xpathObj, 0);
  89 
  90         abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
  91         goto bail;
  92 
  93     }
  94     freeXpathObject(xpathObj);
  95 
  96     // Check for lrm_resource entries
  97     xpathObj = xpath_search(diff,
  98                             "//" F_CIB_UPDATE_RESULT
  99                             "//" XML_TAG_DIFF_ADDED
 100                             "//" XML_LRM_TAG_RESOURCE);
 101     max = numXpathResults(xpathObj);
 102 
 103     /*
 104      * Updates by, or in response to, graph actions will never affect more than
 105      * one resource at a time, so such updates indicate an LRM refresh. In that
 106      * case, start a new transition rather than check each result individually,
 107      * which can result in _huge_ speedups in large clusters.
 108      *
 109      * Unfortunately, we can only do so when there are no pending actions.
 110      * Otherwise, we could mistakenly throw away those results here, and
 111      * the cluster will stall waiting for them and time out the operation.
 112      */
 113     if ((transition_graph->pending == 0) && (max > 1)) {
 114         crm_debug("Ignoring resource operation updates due to history refresh of %d resources",
 115                   max);
 116         crm_log_xml_trace(diff, "lrm-refresh");
 117         abort_transition(INFINITY, tg_restart, "History refresh", NULL);
 118         goto bail;
 119     }
 120 
 121     if (max == 1) {
 122         xmlNode *lrm_resource = getXpathResult(xpathObj, 0);
 123 
 124         if (shutdown_lock_cleared(lrm_resource)) {
 125             // @TODO would be more efficient to abort once after transition done
 126             abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
 127                              lrm_resource);
 128             // Still process results, so we stop timers and update failcounts
 129         }
 130     }
 131     freeXpathObject(xpathObj);
 132 
 133     /* Process operation updates */
 134     xpathObj =
 135         xpath_search(diff,
 136                      "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
 137     max = numXpathResults(xpathObj);
 138     if (max > 0) {
 139         int lpc = 0;
 140 
 141         for (lpc = 0; lpc < max; lpc++) {
 142             xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 143             const char *node = get_node_id(rsc_op);
 144 
 145             process_graph_event(rsc_op, node);
 146         }
 147     }
 148     freeXpathObject(xpathObj);
 149 
 150     /* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
 151     xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
 152     max = numXpathResults(xpathObj);
 153     for (lpc = 0; lpc < max; lpc++) {
 154         int path_max = 0;
 155         const char *op_id = NULL;
 156         char *rsc_op_xpath = NULL;
 157         xmlXPathObject *op_match = NULL;
 158         xmlNode *match = getXpathResult(xpathObj, lpc);
 159 
 160         CRM_LOG_ASSERT(match != NULL);
 161         if(match == NULL) { continue; };
 162 
 163         op_id = ID(match);
 164 
 165         path_max = strlen(RSC_OP_TEMPLATE) + strlen(op_id) + 1;
 166         rsc_op_xpath = calloc(1, path_max);
 167         snprintf(rsc_op_xpath, path_max, RSC_OP_TEMPLATE, op_id);
 168 
 169         op_match = xpath_search(diff, rsc_op_xpath);
 170         if (numXpathResults(op_match) == 0) {
 171             /* Prevent false positives by matching cancelations too */
 172             const char *node = get_node_id(match);
 173             crm_action_t *cancelled = get_cancel_action(op_id, node);
 174 
 175             if (cancelled == NULL) {
 176                 crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
 177                           node);
 178                 abort_transition(INFINITY, tg_restart, "Resource op removal", match);
 179                 freeXpathObject(op_match);
 180                 free(rsc_op_xpath);
 181                 goto bail;
 182 
 183             } else {
 184                 crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
 185                           op_id, node, cancelled->id);
 186             }
 187         }
 188 
 189         freeXpathObject(op_match);
 190         free(rsc_op_xpath);
 191     }
 192 
 193   bail:
 194     freeXpathObject(xpathObj);
 195 }
 196 
 197 static void
 198 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
 199 {
 200     for (xmlNode *rsc_op = pcmk__xml_first_child(lrm_resource); rsc_op != NULL;
 201          rsc_op = pcmk__xml_next(rsc_op)) {
 202         process_graph_event(rsc_op, node);
 203     }
 204     if (shutdown_lock_cleared(lrm_resource)) {
 205         // @TODO would be more efficient to abort once after transition done
 206         abort_transition(INFINITY, tg_restart, "Shutdown lock cleared",
 207                          lrm_resource);
 208     }
 209 }
 210 
 211 static void
 212 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
 213                          const char *op, const char *xpath)
 214 {
 215     xmlNode *rsc = NULL;
 216 
 217     if (xml == NULL) {
 218         return;
 219     }
 220 
 221     if (strcmp(TYPE(xml), XML_CIB_TAG_LRM) == 0) {
 222         xml = first_named_child(xml, XML_LRM_TAG_RESOURCES);
 223         CRM_CHECK(xml != NULL, return);
 224     }
 225 
 226     CRM_CHECK(strcmp(TYPE(xml), XML_LRM_TAG_RESOURCES) == 0, return);
 227 
 228     /*
 229      * Updates by, or in response to, TE actions will never contain updates
 230      * for more than one resource at a time, so such updates indicate an
 231      * LRM refresh.
 232      *
 233      * In that case, start a new transition rather than check each result
 234      * individually, which can result in _huge_ speedups in large clusters.
 235      *
 236      * Unfortunately, we can only do so when there are no pending actions.
 237      * Otherwise, we could mistakenly throw away those results here, and
 238      * the cluster will stall waiting for them and time out the operation.
 239      */
 240     if ((transition_graph->pending == 0)
 241         && xml->children && xml->children->next) {
 242 
 243         crm_log_xml_trace(change, "lrm-refresh");
 244         abort_transition(INFINITY, tg_restart, "History refresh", NULL);
 245         return;
 246     }
 247 
 248     for (rsc = pcmk__xml_first_child(xml); rsc != NULL;
 249          rsc = pcmk__xml_next(rsc)) {
 250         crm_trace("Processing %s", ID(rsc));
 251         process_lrm_resource_diff(rsc, node);
 252     }
 253 }
 254 
 255 static char *extract_node_uuid(const char *xpath) 
     /*  */
 256 {
 257     char *mutable_path = strdup(xpath);
 258     char *node_uuid = NULL;
 259     char *search = NULL;
 260     char *match = NULL;
 261 
 262     match = strstr(mutable_path, "node_state[@id=\'");
 263     if (match == NULL) {
 264         free(mutable_path);
 265         return NULL;
 266     }
 267     match += strlen("node_state[@id=\'");
 268 
 269     search = strchr(match, '\'');
 270     if (search == NULL) {
 271         free(mutable_path);
 272         return NULL;
 273     }
 274     search[0] = 0;
 275 
 276     node_uuid = strdup(match);
 277     free(mutable_path);
 278     return node_uuid;
 279 }
 280 
 281 static void
 282 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 283                   const char *reason)
 284 {
 285     char *node_uuid = NULL;
 286     crm_action_t *down = NULL;
 287 
 288     if(!pcmk__str_eq(op, "delete", pcmk__str_casei)) {
 289         abort_transition(INFINITY, tg_restart, reason, change);
 290         return;
 291     }
 292 
 293     node_uuid = extract_node_uuid(xpath);
 294     if(node_uuid == NULL) {
 295         crm_err("Could not extract node ID from %s", xpath);
 296         abort_transition(INFINITY, tg_restart, reason, change);
 297         return;
 298     }
 299 
 300     down = match_down_event(node_uuid);
 301     if (down == NULL) {
 302         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 303         abort_transition(INFINITY, tg_restart, reason, change);
 304     } else {
 305         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 306     }
 307     free(node_uuid);
 308 }
 309 
 310 static void
 311 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 312 {
 313     char *mutable_key = strdup(xpath);
 314     char *key;
 315     char *node_uuid;
 316 
 317     // Extract the part of xpath between last pair of single quotes
 318     key = strrchr(mutable_key, '\'');
 319     if (key != NULL) {
 320         *key = '\0';
 321         key = strrchr(mutable_key, '\'');
 322     }
 323     if (key == NULL) {
 324         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 325                  xpath);
 326         free(mutable_key);
 327         return;
 328     }
 329     ++key;
 330 
 331     node_uuid = extract_node_uuid(xpath);
 332     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 333         abort_transition(INFINITY, tg_restart, "Resource operation removal",
 334                          change);
 335     }
 336     free(mutable_key);
 337     free(node_uuid);
 338 }
 339 
 340 static void
 341 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 342 {
 343     if (strstr(xpath, "/" XML_LRM_TAG_RSC_OP "[")) {
 344         process_op_deletion(xpath, change);
 345 
 346     } else if (strstr(xpath, "/" XML_CIB_TAG_LRM "[")) {
 347         abort_unless_down(xpath, op, change, "Resource state removal");
 348 
 349     } else if (strstr(xpath, "/" XML_CIB_TAG_STATE "[")) {
 350         abort_unless_down(xpath, op, change, "Node state removal");
 351 
 352     } else {
 353         crm_trace("Ignoring delete of %s", xpath);
 354     }
 355 }
 356 
 357 static void
 358 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 359                         const char *xpath)
 360 {
 361     xmlNode *lrm = first_named_child(state, XML_CIB_TAG_LRM);
 362 
 363     process_resource_updates(ID(state), lrm, change, op, xpath);
 364 }
 365 
 366 static void
 367 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 368                     const char *xpath)
 369 {
 370     for (xmlNode *state = pcmk__xml_first_child(status); state != NULL;
 371          state = pcmk__xml_next(state)) {
 372         process_node_state_diff(state, change, op, xpath);
 373     }
 374 }
 375 
 376 static void
 377 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 378                  const char *xpath)
 379 {
 380     xmlNode *status = first_named_child(cib, XML_CIB_TAG_STATUS);
 381     xmlNode *config = first_named_child(cib, XML_CIB_TAG_CONFIGURATION);
 382 
 383     if (status) {
 384         process_status_diff(status, change, op, xpath);
 385     }
 386     if (config) {
 387         abort_transition(INFINITY, tg_restart,
 388                          "Non-status-only change", change);
 389     }
 390 }
 391 
 392 static void
 393 te_update_diff_v2(xmlNode *diff)
     /*  */
 394 {
 395     crm_log_xml_trace(diff, "Patch:Raw");
 396 
 397     for (xmlNode *change = pcmk__xml_first_child(diff); change != NULL;
 398          change = pcmk__xml_next(change)) {
 399 
 400         xmlNode *match = NULL;
 401         const char *name = NULL;
 402         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
 403 
 404         // Possible ops: create, modify, delete, move
 405         const char *op = crm_element_value(change, XML_DIFF_OP);
 406 
 407         // Ignore uninteresting updates
 408         if (op == NULL) {
 409             continue;
 410 
 411         } else if (xpath == NULL) {
 412             crm_trace("Ignoring %s change for version field", op);
 413             continue;
 414 
 415         } else if (strcmp(op, "move") == 0) {
 416             crm_trace("Ignoring move change at %s", xpath);
 417             continue;
 418         }
 419 
 420         // Find the result of create/modify ops
 421         if (strcmp(op, "create") == 0) {
 422             match = change->children;
 423 
 424         } else if (strcmp(op, "modify") == 0) {
 425             match = first_named_child(change, XML_DIFF_RESULT);
 426             if(match) {
 427                 match = match->children;
 428             }
 429 
 430         } else if (strcmp(op, "delete") != 0) {
 431             crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 432                      op, xpath);
 433             continue;
 434         }
 435 
 436         if (match) {
 437             if (match->type == XML_COMMENT_NODE) {
 438                 crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 439                 continue;
 440             }
 441             name = (const char *)match->name;
 442         }
 443 
 444         crm_trace("Handling %s operation for %s%s%s",
 445                   op, (xpath? xpath : "CIB"),
 446                   (name? " matched by " : ""), (name? name : ""));
 447 
 448         if (strstr(xpath, "/" XML_TAG_CIB "/" XML_CIB_TAG_CONFIGURATION)) {
 449             abort_transition(INFINITY, tg_restart, "Configuration change",
 450                              change);
 451             break; // Won't be packaged with operation results we may be waiting for
 452 
 453         } else if (strstr(xpath, "/" XML_CIB_TAG_TICKETS)
 454                    || pcmk__str_eq(name, XML_CIB_TAG_TICKETS, pcmk__str_casei)) {
 455             abort_transition(INFINITY, tg_restart, "Ticket attribute change", change);
 456             break; // Won't be packaged with operation results we may be waiting for
 457 
 458         } else if (strstr(xpath, "/" XML_TAG_TRANSIENT_NODEATTRS "[")
 459                    || pcmk__str_eq(name, XML_TAG_TRANSIENT_NODEATTRS, pcmk__str_casei)) {
 460             abort_unless_down(xpath, op, change, "Transient attribute change");
 461             break; // Won't be packaged with operation results we may be waiting for
 462 
 463         } else if (strcmp(op, "delete") == 0) {
 464             process_delete_diff(xpath, op, change);
 465 
 466         } else if (name == NULL) {
 467             crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 468                      op, xpath);
 469 
 470         } else if (strcmp(name, XML_TAG_CIB) == 0) {
 471             process_cib_diff(match, change, op, xpath);
 472 
 473         } else if (strcmp(name, XML_CIB_TAG_STATUS) == 0) {
 474             process_status_diff(match, change, op, xpath);
 475 
 476         } else if (strcmp(name, XML_CIB_TAG_STATE) == 0) {
 477             process_node_state_diff(match, change, op, xpath);
 478 
 479         } else if (strcmp(name, XML_CIB_TAG_LRM) == 0) {
 480             process_resource_updates(ID(match), match, change, op, xpath);
 481 
 482         } else if (strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
 483             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 484 
 485             process_resource_updates(local_node, match, change, op, xpath);
 486             free(local_node);
 487 
 488         } else if (strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
 489             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 490 
 491             process_lrm_resource_diff(match, local_node);
 492             free(local_node);
 493 
 494         } else if (strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
 495             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
 496 
 497             process_graph_event(match, local_node);
 498             free(local_node);
 499 
 500         } else {
 501             crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 502                      op, xpath, name);
 503         }
 504     }
 505 }
 506 
 507 void
 508 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 509 {
 510     xmlNode *diff = NULL;
 511     const char *op = NULL;
 512     int rc = -EINVAL;
 513     int format = 1;
 514     int p_add[] = { 0, 0, 0 };
 515     int p_del[] = { 0, 0, 0 };
 516 
 517     CRM_CHECK(msg != NULL, return);
 518     crm_element_value_int(msg, F_CIB_RC, &rc);
 519 
 520     if (transition_graph == NULL) {
 521         crm_trace("No graph");
 522         return;
 523 
 524     } else if (rc < pcmk_ok) {
 525         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 526         return;
 527 
 528     } else if (transition_graph->complete
 529                && fsa_state != S_IDLE
 530                && fsa_state != S_TRANSITION_ENGINE
 531                && fsa_state != S_POLICY_ENGINE) {
 532         crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
 533                   transition_graph->complete);
 534         return;
 535     }
 536 
 537     op = crm_element_value(msg, F_CIB_OPERATION);
 538     diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
 539 
 540     xml_patch_versions(diff, p_add, p_del);
 541     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 542               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 543               fsa_state2string(fsa_state));
 544 
 545     crm_element_value_int(diff, "format", &format);
 546     switch (format) {
 547         case 1:
 548             te_update_diff_v1(event, diff);
 549             break;
 550         case 2:
 551             te_update_diff_v2(diff);
 552             break;
 553         default:
 554             crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 555                      format);
 556     }
 557 }
 558 
 559 gboolean
 560 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 561 {
 562     const char *from = crm_element_value(msg, F_ORIG);
 563     const char *sys_to = crm_element_value(msg, F_CRM_SYS_TO);
 564     const char *sys_from = crm_element_value(msg, F_CRM_SYS_FROM);
 565     const char *ref = crm_element_value(msg, F_CRM_REFERENCE);
 566     const char *op = crm_element_value(msg, F_CRM_TASK);
 567     const char *type = crm_element_value(msg, F_CRM_MSG_TYPE);
 568 
 569     crm_trace("Processing %s (%s) message", op, ref);
 570     crm_log_xml_trace(msg, "ipc");
 571 
 572     if (op == NULL) {
 573         /* error */
 574 
 575     } else if (sys_to == NULL || strcasecmp(sys_to, CRM_SYSTEM_TENGINE) != 0) {
 576         crm_trace("Bad sys-to %s", crm_str(sys_to));
 577         return FALSE;
 578 
 579     } else if (pcmk__str_eq(op, CRM_OP_INVOKE_LRM, pcmk__str_casei)
 580                && pcmk__str_eq(sys_from, CRM_SYSTEM_LRMD, pcmk__str_casei)
 581 /*                && pcmk__str_eq(type, XML_ATTR_RESPONSE, pcmk__str_casei) */
 582         ) {
 583         xmlXPathObject *xpathObj = NULL;
 584 
 585         crm_log_xml_trace(msg, "Processing (N)ACK");
 586         crm_debug("Processing (N)ACK %s from %s", crm_element_value(msg, F_CRM_REFERENCE), from);
 587 
 588         xpathObj = xpath_search(xml_data, "//" XML_LRM_TAG_RSC_OP);
 589         if (numXpathResults(xpathObj)) {
 590             int lpc = 0, max = numXpathResults(xpathObj);
 591 
 592             for (lpc = 0; lpc < max; lpc++) {
 593                 xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
 594                 const char *node = get_node_id(rsc_op);
 595 
 596                 process_graph_event(rsc_op, node);
 597             }
 598             freeXpathObject(xpathObj);
 599 
 600         } else {
 601             crm_log_xml_err(msg, "Invalid (N)ACK");
 602             freeXpathObject(xpathObj);
 603             return FALSE;
 604         }
 605 
 606     } else {
 607         crm_err("Unknown command: %s::%s from %s", type, op, sys_from);
 608     }
 609 
 610     crm_trace("finished processing message");
 611 
 612     return TRUE;
 613 }
 614 
 615 void
 616 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 617 {
 618     if (rc < pcmk_ok) {
 619         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 620     }
 621 }
 622 
 623 /*!
 624  * \brief Handle a timeout in node-to-node communication
 625  *
 626  * \param[in] data  Pointer to action timer
 627  *
 628  * \return FALSE (indicating that source should be not be re-added)
 629  */
 630 gboolean
 631 action_timer_callback(gpointer data)
     /*  */
 632 {
 633     crm_action_timer_t *timer = NULL;
 634     const char *task = NULL;
 635     const char *on_node = NULL;
 636     const char *via_node = NULL;
 637 
 638     CRM_CHECK(data != NULL, return FALSE);
 639 
 640     timer = (crm_action_timer_t *) data;
 641     stop_te_timer(timer);
 642 
 643     CRM_CHECK(timer->action != NULL, return FALSE);
 644 
 645     task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
 646     on_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_TARGET);
 647     via_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_ROUTER_NODE);
 648 
 649     if (transition_graph->complete) {
 650         crm_notice("Node %s did not send %s result (via %s) within %dms "
 651                    "(ignoring because transition not in progress)",
 652                    (on_node? on_node : ""), (task? task : "unknown action"),
 653                    (via_node? via_node : "controller"), timer->timeout);
 654     } else {
 655         /* fail the action */
 656 
 657         crm_err("Node %s did not send %s result (via %s) within %dms "
 658                 "(action timeout plus cluster-delay)",
 659                 (on_node? on_node : ""), (task? task : "unknown action"),
 660                 (via_node? via_node : "controller"),
 661                 timer->timeout + transition_graph->network_delay);
 662         pcmk__log_graph_action(LOG_ERR, timer->action);
 663 
 664         crm__set_graph_action_flags(timer->action, pcmk__graph_action_failed);
 665 
 666         te_action_confirmed(timer->action, transition_graph);
 667         abort_transition(INFINITY, tg_restart, "Action lost", NULL);
 668 
 669         // Record timeout in the CIB if appropriate
 670         if ((timer->action->type == action_type_rsc)
 671             && controld_action_is_recordable(task)) {
 672             controld_record_action_timeout(timer->action);
 673         }
 674     }
 675 
 676     return FALSE;
 677 }

/* */