daemons/controld/controld_te

/* */
This source file includes following definitions.
shutdown_lock_cleared
process_lrm_resource_diff
process_resource_updates
extract_node_uuid
abort_unless_down
process_op_deletion
process_delete_diff
process_node_state_diff
process_status_diff
process_cib_diff
te_update_diff_element
te_update_diff
process_te_message
cib_action_updated
action_timer_callback
   1 /*
   2  * Copyright 2004-2025 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/stat.h>
  13 
  14 #include <crm/crm.h>
  15 #include <crm/common/xml.h>
  16 #include <crm/common/xml_internal.h>
  17 
  18 #include <libxml/xpath.h>               // xmlXPathObject, etc.
  19 
  20 #include <pacemaker-controld.h>
  21 
  22 // An explicit PCMK_OPT_SHUTDOWN_LOCK of 0 means the lock has been cleared
  23 static bool
  24 shutdown_lock_cleared(xmlNode *lrm_resource)
     /*  */
  25 {
  26     time_t shutdown_lock = 0;
  27 
  28     return (crm_element_value_epoch(lrm_resource, PCMK_OPT_SHUTDOWN_LOCK,
  29                                     &shutdown_lock) == pcmk_ok)
  30            && (shutdown_lock == 0);
  31 }
  32 
  33 static void
  34 process_lrm_resource_diff(xmlNode *lrm_resource, const char *node)
     /*  */
  35 {
  36     for (xmlNode *rsc_op = pcmk__xe_first_child(lrm_resource, NULL, NULL, NULL);
  37          rsc_op != NULL; rsc_op = pcmk__xe_next(rsc_op, NULL)) {
  38         process_graph_event(rsc_op, node);
  39     }
  40     if (shutdown_lock_cleared(lrm_resource)) {
  41         // @TODO would be more efficient to abort once after transition done
  42         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  43                          "Shutdown lock cleared", lrm_resource);
  44     }
  45 }
  46 
  47 static void
  48 process_resource_updates(const char *node, xmlNode *xml, xmlNode *change,
     /*  */
  49                          const char *op, const char *xpath)
  50 {
  51     xmlNode *rsc = NULL;
  52 
  53     if (xml == NULL) {
  54         return;
  55     }
  56 
  57     if (pcmk__xe_is(xml, PCMK__XE_LRM)) {
  58         xml = pcmk__xe_first_child(xml, PCMK__XE_LRM_RESOURCES, NULL, NULL);
  59         CRM_CHECK(xml != NULL, return);
  60     }
  61 
  62     CRM_CHECK(pcmk__xe_is(xml, PCMK__XE_LRM_RESOURCES), return);
  63 
  64     /*
  65      * Updates by, or in response to, TE actions will never contain updates
  66      * for more than one resource at a time, so such updates indicate an
  67      * LRM refresh.
  68      *
  69      * In that case, start a new transition rather than check each result
  70      * individually, which can result in _huge_ speedups in large clusters.
  71      *
  72      * Unfortunately, we can only do so when there are no pending actions.
  73      * Otherwise, we could mistakenly throw away those results here, and
  74      * the cluster will stall waiting for them and time out the operation.
  75      */
  76     if ((controld_globals.transition_graph->pending == 0)
  77         && (xml->children != NULL) && (xml->children->next != NULL)) {
  78 
  79         crm_log_xml_trace(change, "lrm-refresh");
  80         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
  81                          "History refresh", NULL);
  82         return;
  83     }
  84 
  85     for (rsc = pcmk__xe_first_child(xml, NULL, NULL, NULL); rsc != NULL;
  86          rsc = pcmk__xe_next(rsc, NULL)) {
  87         crm_trace("Processing %s", pcmk__xe_id(rsc));
  88         process_lrm_resource_diff(rsc, node);
  89     }
  90 }
  91 
  92 static char *extract_node_uuid(const char *xpath) 
     /*  */
  93 {
  94     char *mutable_path = pcmk__str_copy(xpath);
  95     char *node_uuid = NULL;
  96     char *search = NULL;
  97     char *match = NULL;
  98 
  99     match = strstr(mutable_path, PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");
 100     if (match == NULL) {
 101         free(mutable_path);
 102         return NULL;
 103     }
 104     match += strlen(PCMK__XE_NODE_STATE "[@" PCMK_XA_ID "=\'");
 105 
 106     search = strchr(match, '\'');
 107     if (search == NULL) {
 108         free(mutable_path);
 109         return NULL;
 110     }
 111     search[0] = 0;
 112 
 113     node_uuid = pcmk__str_copy(match);
 114     free(mutable_path);
 115     return node_uuid;
 116 }
 117 
 118 static void
 119 abort_unless_down(const char *xpath, const char *op, xmlNode *change,
     /*  */
 120                   const char *reason)
 121 {
 122     char *node_uuid = NULL;
 123     pcmk__graph_action_t *down = NULL;
 124 
 125     if (!pcmk__str_eq(op, PCMK_VALUE_DELETE, pcmk__str_none)) {
 126         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 127                          change);
 128         return;
 129     }
 130 
 131     node_uuid = extract_node_uuid(xpath);
 132     if(node_uuid == NULL) {
 133         crm_err("Could not extract node ID from %s", xpath);
 134         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 135                          change);
 136         return;
 137     }
 138 
 139     down = match_down_event(node_uuid);
 140     if (down == NULL) {
 141         crm_trace("Not expecting %s to be down (%s)", node_uuid, xpath);
 142         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart, reason,
 143                          change);
 144     } else {
 145         crm_trace("Expecting changes to %s (%s)", node_uuid, xpath);
 146     }
 147     free(node_uuid);
 148 }
 149 
 150 static void
 151 process_op_deletion(const char *xpath, xmlNode *change)
     /*  */
 152 {
 153     char *mutable_key = pcmk__str_copy(xpath);
 154     char *key;
 155     char *node_uuid;
 156 
 157     // Extract the part of xpath between last pair of single quotes
 158     key = strrchr(mutable_key, '\'');
 159     if (key != NULL) {
 160         *key = '\0';
 161         key = strrchr(mutable_key, '\'');
 162     }
 163     if (key == NULL) {
 164         crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
 165                  xpath);
 166         free(mutable_key);
 167         return;
 168     }
 169     ++key;
 170 
 171     node_uuid = extract_node_uuid(xpath);
 172     if (confirm_cancel_action(key, node_uuid) == FALSE) {
 173         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 174                          "Resource operation removal", change);
 175     }
 176     free(mutable_key);
 177     free(node_uuid);
 178 }
 179 
 180 static void
 181 process_delete_diff(const char *xpath, const char *op, xmlNode *change)
     /*  */
 182 {
 183     if (strstr(xpath, "/" PCMK__XE_LRM_RSC_OP "[")) {
 184         process_op_deletion(xpath, change);
 185 
 186     } else if (strstr(xpath, "/" PCMK__XE_LRM "[")) {
 187         abort_unless_down(xpath, op, change, "Resource state removal");
 188 
 189     } else if (strstr(xpath, "/" PCMK__XE_NODE_STATE "[")) {
 190         abort_unless_down(xpath, op, change, "Node state removal");
 191 
 192     } else {
 193         crm_trace("Ignoring delete of %s", xpath);
 194     }
 195 }
 196 
 197 static void
 198 process_node_state_diff(xmlNode *state, xmlNode *change, const char *op,
     /*  */
 199                         const char *xpath)
 200 {
 201     xmlNode *lrm = pcmk__xe_first_child(state, PCMK__XE_LRM, NULL, NULL);
 202 
 203     process_resource_updates(pcmk__xe_id(state), lrm, change, op, xpath);
 204 }
 205 
 206 static void
 207 process_status_diff(xmlNode *status, xmlNode *change, const char *op,
     /*  */
 208                     const char *xpath)
 209 {
 210     for (xmlNode *state = pcmk__xe_first_child(status, NULL, NULL, NULL);
 211          state != NULL; state = pcmk__xe_next(state, NULL)) {
 212 
 213         process_node_state_diff(state, change, op, xpath);
 214     }
 215 }
 216 
 217 static void
 218 process_cib_diff(xmlNode *cib, xmlNode *change, const char *op,
     /*  */
 219                  const char *xpath)
 220 {
 221     xmlNode *status = pcmk__xe_first_child(cib, PCMK_XE_STATUS, NULL, NULL);
 222     xmlNode *config = pcmk__xe_first_child(cib, PCMK_XE_CONFIGURATION, NULL,
 223                                            NULL);
 224 
 225     if (status) {
 226         process_status_diff(status, change, op, xpath);
 227     }
 228     if (config) {
 229         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 230                          "Non-status-only change", change);
 231     }
 232 }
 233 
 234 static int
 235 te_update_diff_element(xmlNode *change, void *userdata)
     /*  */
 236 {
 237     xmlNode *match = NULL;
 238     const char *name = NULL;
 239     const char *xpath = crm_element_value(change, PCMK_XA_PATH);
 240 
 241     // Possible ops: create, modify, delete, move
 242     const char *op = crm_element_value(change, PCMK_XA_OPERATION);
 243 
 244     // Ignore uninteresting updates
 245     if (op == NULL) {
 246         return pcmk_rc_ok;
 247 
 248     } else if (xpath == NULL) {
 249         crm_trace("Ignoring %s change for version field", op);
 250         return pcmk_rc_ok;
 251 
 252     } else if ((strcmp(op, PCMK_VALUE_MOVE) == 0)
 253                && (strstr(xpath,
 254                           "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION
 255                           "/" PCMK_XE_RESOURCES) == NULL)) {
 256         /* We still need to consider moves within the resources section,
 257          * since they affect placement order.
 258          */
 259         crm_trace("Ignoring move change at %s", xpath);
 260         return pcmk_rc_ok;
 261     }
 262 
 263     // Find the result of create/modify ops
 264     if (strcmp(op, PCMK_VALUE_CREATE) == 0) {
 265         match = change->children;
 266 
 267     } else if (strcmp(op, PCMK_VALUE_MODIFY) == 0) {
 268         match = pcmk__xe_first_child(change, PCMK_XE_CHANGE_RESULT, NULL, NULL);
 269         if(match) {
 270             match = match->children;
 271         }
 272 
 273     } else if (!pcmk__str_any_of(op,
 274                                  PCMK_VALUE_DELETE, PCMK_VALUE_MOVE,
 275                                  NULL)) {
 276         crm_warn("Ignoring malformed CIB update (%s operation on %s is unrecognized)",
 277                  op, xpath);
 278         return pcmk_rc_ok;
 279     }
 280 
 281     if (match) {
 282         if (match->type == XML_COMMENT_NODE) {
 283             crm_trace("Ignoring %s operation for comment at %s", op, xpath);
 284             return pcmk_rc_ok;
 285         }
 286         name = (const char *)match->name;
 287     }
 288 
 289     crm_trace("Handling %s operation for %s%s%s",
 290               op, (xpath? xpath : "CIB"),
 291               (name? " matched by " : ""), (name? name : ""));
 292 
 293     if (strstr(xpath, "/" PCMK_XE_CIB "/" PCMK_XE_CONFIGURATION)) {
 294         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 295                          "Configuration change", change);
 296         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 297 
 298     } else if (strstr(xpath, "/" PCMK_XE_TICKETS)
 299                || pcmk__str_eq(name, PCMK_XE_TICKETS, pcmk__str_none)) {
 300         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 301                          "Ticket attribute change", change);
 302         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 303 
 304     } else if (strstr(xpath, "/" PCMK__XE_TRANSIENT_ATTRIBUTES "[")
 305                || pcmk__str_eq(name, PCMK__XE_TRANSIENT_ATTRIBUTES,
 306                                pcmk__str_none)) {
 307         abort_unless_down(xpath, op, change, "Transient attribute change");
 308         return pcmk_rc_cib_modified; // Won't be packaged with operation results we may be waiting for
 309 
 310     } else if (strcmp(op, PCMK_VALUE_DELETE) == 0) {
 311         process_delete_diff(xpath, op, change);
 312 
 313     } else if (name == NULL) {
 314         crm_warn("Ignoring malformed CIB update (%s at %s has no result)",
 315                  op, xpath);
 316 
 317     } else if (strcmp(name, PCMK_XE_CIB) == 0) {
 318         process_cib_diff(match, change, op, xpath);
 319 
 320     } else if (strcmp(name, PCMK_XE_STATUS) == 0) {
 321         process_status_diff(match, change, op, xpath);
 322 
 323     } else if (strcmp(name, PCMK__XE_NODE_STATE) == 0) {
 324         process_node_state_diff(match, change, op, xpath);
 325 
 326     } else if (strcmp(name, PCMK__XE_LRM) == 0) {
 327         process_resource_updates(pcmk__xe_id(match), match, change, op,
 328                                  xpath);
 329 
 330     } else if (strcmp(name, PCMK__XE_LRM_RESOURCES) == 0) {
 331         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 332 
 333         process_resource_updates(local_node, match, change, op, xpath);
 334         free(local_node);
 335 
 336     } else if (strcmp(name, PCMK__XE_LRM_RESOURCE) == 0) {
 337         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 338 
 339         process_lrm_resource_diff(match, local_node);
 340         free(local_node);
 341 
 342     } else if (strcmp(name, PCMK__XE_LRM_RSC_OP) == 0) {
 343         char *local_node = pcmk__xpath_node_id(xpath, PCMK__XE_LRM);
 344 
 345         process_graph_event(match, local_node);
 346         free(local_node);
 347 
 348     } else {
 349         crm_warn("Ignoring malformed CIB update (%s at %s has unrecognized result %s)",
 350                  op, xpath, name);
 351     }
 352 
 353     return pcmk_rc_ok;
 354 }
 355 
 356 void
 357 te_update_diff(const char *event, xmlNode * msg)
     /*  */
 358 {
 359     xmlNode *wrapper = NULL;
 360     xmlNode *diff = NULL;
 361     const char *op = NULL;
 362     int rc = -EINVAL;
 363     int format = 1;
 364     int p_add[] = { 0, 0, 0 };
 365     int p_del[] = { 0, 0, 0 };
 366 
 367     CRM_CHECK(msg != NULL, return);
 368     crm_element_value_int(msg, PCMK__XA_CIB_RC, &rc);
 369 
 370     if (controld_globals.transition_graph == NULL) {
 371         crm_trace("No graph");
 372         return;
 373 
 374     } else if (rc < pcmk_ok) {
 375         crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
 376         return;
 377 
 378     } else if (controld_globals.transition_graph->complete
 379                && (controld_globals.fsa_state != S_IDLE)
 380                && (controld_globals.fsa_state != S_TRANSITION_ENGINE)
 381                && (controld_globals.fsa_state != S_POLICY_ENGINE)) {
 382         crm_trace("Filter state=%s (complete)",
 383                   fsa_state2string(controld_globals.fsa_state));
 384         return;
 385     }
 386 
 387     op = crm_element_value(msg, PCMK__XA_CIB_OP);
 388 
 389     wrapper = pcmk__xe_first_child(msg, PCMK__XE_CIB_UPDATE_RESULT, NULL, NULL);
 390     diff = pcmk__xe_first_child(wrapper, NULL, NULL, NULL);
 391 
 392     xml_patch_versions(diff, p_add, p_del);
 393     crm_debug("Processing (%s) diff: %d.%d.%d -> %d.%d.%d (%s)", op,
 394               p_del[0], p_del[1], p_del[2], p_add[0], p_add[1], p_add[2],
 395               fsa_state2string(controld_globals.fsa_state));
 396 
 397     crm_element_value_int(diff, PCMK_XA_FORMAT, &format);
 398 
 399     if (format == 2) {
 400         crm_log_xml_trace(diff, "patch");
 401         pcmk__xe_foreach_child(diff, NULL, te_update_diff_element, NULL);
 402 
 403     } else {
 404         crm_warn("Ignoring malformed CIB update (unknown patch format %d)",
 405                  format);
 406     }
 407     controld_remove_all_outside_events();
 408 }
 409 
 410 void
 411 process_te_message(xmlNode * msg, xmlNode * xml_data)
     /*  */
 412 {
 413     const char *value = NULL;
 414     xmlXPathObject *xpathObj = NULL;
 415     int nmatches = 0;
 416 
 417     CRM_CHECK(msg != NULL, return);
 418 
 419     // Transition requests must specify transition engine as subsystem
 420     value = crm_element_value(msg, PCMK__XA_CRM_SYS_TO);
 421     if (pcmk__str_empty(value)
 422         || !pcmk__str_eq(value, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
 423         crm_info("Received invalid transition request: subsystem '%s' not '"
 424                  CRM_SYSTEM_TENGINE "'", pcmk__s(value, ""));
 425         return;
 426     }
 427 
 428     // Only the lrm_invoke command is supported as a transition request
 429     value = crm_element_value(msg, PCMK__XA_CRM_TASK);
 430     if (!pcmk__str_eq(value, CRM_OP_INVOKE_LRM, pcmk__str_none)) {
 431         crm_info("Received invalid transition request: command '%s' not '"
 432                  CRM_OP_INVOKE_LRM "'", pcmk__s(value, ""));
 433         return;
 434     }
 435 
 436     // Transition requests must be marked as coming from the executor
 437     value = crm_element_value(msg, PCMK__XA_CRM_SYS_FROM);
 438     if (!pcmk__str_eq(value, CRM_SYSTEM_LRMD, pcmk__str_none)) {
 439         crm_info("Received invalid transition request: from '%s' not '"
 440                  CRM_SYSTEM_LRMD "'", pcmk__s(value, ""));
 441         return;
 442     }
 443 
 444     crm_debug("Processing transition request with ref='%s' origin='%s'",
 445               pcmk__s(crm_element_value(msg, PCMK_XA_REFERENCE), ""),
 446               pcmk__s(crm_element_value(msg, PCMK__XA_SRC), ""));
 447 
 448     xpathObj = pcmk__xpath_search(xml_data->doc, "//" PCMK__XE_LRM_RSC_OP);
 449     nmatches = pcmk__xpath_num_results(xpathObj);
 450     if (nmatches == 0) {
 451         crm_err("Received transition request with no results (bug?)");
 452     } else {
 453         for (int lpc = 0; lpc < nmatches; lpc++) {
 454             xmlNode *rsc_op = pcmk__xpath_result(xpathObj, lpc);
 455 
 456             if (rsc_op != NULL) {
 457                 const char *node = get_node_id(rsc_op);
 458 
 459                 process_graph_event(rsc_op, node);
 460             }
 461         }
 462     }
 463     xmlXPathFreeObject(xpathObj);
 464 }
 465 
 466 void
 467 cib_action_updated(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
     /*  */
 468 {
 469     if (rc < pcmk_ok) {
 470         crm_err("Update %d FAILED: %s", call_id, pcmk_strerror(rc));
 471     }
 472 }
 473 
 474 /*!
 475  * \brief Handle a timeout in node-to-node communication
 476  *
 477  * \param[in,out] data  Pointer to graph action
 478  *
 479  * \return FALSE (indicating that source should be not be re-added)
 480  */
 481 gboolean
 482 action_timer_callback(gpointer data)
     /*  */
 483 {
 484     pcmk__graph_action_t *action = (pcmk__graph_action_t *) data;
 485     const char *task = NULL;
 486     const char *on_node = NULL;
 487     const char *via_node = NULL;
 488 
 489     CRM_CHECK(data != NULL, return FALSE);
 490 
 491     stop_te_timer(action);
 492 
 493     task = crm_element_value(action->xml, PCMK_XA_OPERATION);
 494     on_node = crm_element_value(action->xml, PCMK__META_ON_NODE);
 495     via_node = crm_element_value(action->xml, PCMK__XA_ROUTER_NODE);
 496 
 497     if (controld_globals.transition_graph->complete) {
 498         crm_notice("Node %s did not send %s result (via %s) within %dms "
 499                    "(ignoring because transition not in progress)",
 500                    (on_node? on_node : ""), (task? task : "unknown action"),
 501                    (via_node? via_node : "controller"), action->timeout);
 502     } else {
 503         /* fail the action */
 504 
 505         crm_err("Node %s did not send %s result (via %s) within %dms "
 506                 "(action timeout plus " PCMK_OPT_CLUSTER_DELAY ")",
 507                 (on_node? on_node : ""), (task? task : "unknown action"),
 508                 (via_node? via_node : "controller"),
 509                 (action->timeout
 510                  + controld_globals.transition_graph->network_delay));
 511         pcmk__log_graph_action(LOG_ERR, action);
 512 
 513         pcmk__set_graph_action_flags(action, pcmk__graph_action_failed);
 514 
 515         te_action_confirmed(action, controld_globals.transition_graph);
 516         abort_transition(PCMK_SCORE_INFINITY, pcmk__graph_restart,
 517                          "Action lost", NULL);
 518 
 519         // Record timeout in the CIB if appropriate
 520         if ((action->type == pcmk__rsc_graph_action)
 521             && controld_action_is_recordable(task)) {
 522             controld_record_action_timeout(action);
 523         }
 524     }
 525 
 526     return FALSE;
 527 }
/* */
root/daemons/controld/controld_te_callbacks.c

DEFINITIONS
