root/lib/pacemaker/pcmk_sched_transition.c

/* [previous][next][first][last][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. inject_transient_attr
  2. update_failcounts
  3. create_node_entry
  4. create_op
  5. inject_op
  6. inject_node_state
  7. modify_node
  8. find_resource_xml
  9. inject_resource
  10. find_ticket_state
  11. set_ticket_state_attr
  12. modify_configuration
  13. exec_pseudo_action
  14. exec_rsc_action
  15. exec_crmd_action
  16. exec_stonith_action
  17. run_simulation

   1 /*
   2  * Copyright 2009-2021 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <unistd.h>
  14 #include <stdlib.h>
  15 
  16 #include <sys/stat.h>
  17 #include <sys/param.h>
  18 #include <sys/types.h>
  19 #include <dirent.h>
  20 
  21 #include <crm/crm.h>
  22 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_free_event()
  23 #include <crm/cib.h>
  24 #include <crm/common/util.h>
  25 #include <crm/common/iso8601.h>
  26 #include <crm/common/xml_internal.h>
  27 #include <crm/lrmd_internal.h>
  28 #include <crm/pengine/status.h>
  29 #include <pacemaker-internal.h>
  30 
  31 static pcmk__output_t *out = NULL;
  32 static cib_t *fake_cib = NULL;
  33 static GList *fake_resource_list = NULL;
  34 static GList *fake_op_fail_list = NULL;
  35 gboolean bringing_nodes_online = FALSE;
  36 
  37 #define STATUS_PATH_MAX 512
  38 
  39 #define NEW_NODE_TEMPLATE "//"XML_CIB_TAG_NODE"[@uname='%s']"
  40 #define NODE_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']"
  41 #define RSC_TEMPLATE "//"XML_CIB_TAG_STATE"[@uname='%s']//"XML_LRM_TAG_RESOURCE"[@id='%s']"
  42 
  43 
  44 static void
  45 inject_transient_attr(xmlNode * cib_node, const char *name, const char *value)
     /* [previous][next][first][last][top][bottom][index][help] */
  46 {
  47     xmlNode *attrs = NULL;
  48     xmlNode *instance_attrs = NULL;
  49     const char *node_uuid = ID(cib_node);
  50 
  51     out->message(out, "inject-attr", name, value, cib_node);
  52 
  53     attrs = first_named_child(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
  54     if (attrs == NULL) {
  55         attrs = create_xml_node(cib_node, XML_TAG_TRANSIENT_NODEATTRS);
  56         crm_xml_add(attrs, XML_ATTR_ID, node_uuid);
  57     }
  58 
  59     instance_attrs = first_named_child(attrs, XML_TAG_ATTR_SETS);
  60     if (instance_attrs == NULL) {
  61         instance_attrs = create_xml_node(attrs, XML_TAG_ATTR_SETS);
  62         crm_xml_add(instance_attrs, XML_ATTR_ID, node_uuid);
  63     }
  64 
  65     crm_create_nvpair_xml(instance_attrs, NULL, name, value);
  66 }
  67 
  68 static void
  69 update_failcounts(xmlNode * cib_node, const char *resource, const char *task,
     /* [previous][next][first][last][top][bottom][index][help] */
  70                   guint interval_ms, int rc)
  71 {
  72     if (rc == 0) {
  73         return;
  74 
  75     } else if ((rc == 7) && (interval_ms == 0)) {
  76         return;
  77 
  78     } else {
  79         char *name = NULL;
  80         char *now = pcmk__ttoa(time(NULL));
  81 
  82         name = pcmk__failcount_name(resource, task, interval_ms);
  83         inject_transient_attr(cib_node, name, "value++");
  84         free(name);
  85 
  86         name = pcmk__lastfailure_name(resource, task, interval_ms);
  87         inject_transient_attr(cib_node, name, now);
  88         free(name);
  89         free(now);
  90     }
  91 }
  92 
  93 static void
  94 create_node_entry(cib_t * cib_conn, const char *node)
     /* [previous][next][first][last][top][bottom][index][help] */
  95 {
  96     int rc = pcmk_ok;
  97     char *xpath = crm_strdup_printf(NEW_NODE_TEMPLATE, node);
  98 
  99     rc = cib_conn->cmds->query(cib_conn, xpath, NULL, cib_xpath | cib_sync_call | cib_scope_local);
 100 
 101     if (rc == -ENXIO) {
 102         xmlNode *cib_object = create_xml_node(NULL, XML_CIB_TAG_NODE);
 103 
 104         crm_xml_add(cib_object, XML_ATTR_ID, node); // Use node name as ID
 105         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
 106         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_NODES, cib_object,
 107                                cib_sync_call | cib_scope_local);
 108         /* Not bothering with subsequent query to see if it exists,
 109            we'll bomb out later in the call to query_node_uuid()... */
 110 
 111         free_xml(cib_object);
 112     }
 113 
 114     free(xpath);
 115 }
 116 
 117 static lrmd_event_data_t *
 118 create_op(xmlNode *cib_resource, const char *task, guint interval_ms,
     /* [previous][next][first][last][top][bottom][index][help] */
 119           int outcome)
 120 {
 121     lrmd_event_data_t *op = NULL;
 122     xmlNode *xop = NULL;
 123 
 124     op = lrmd_new_event(ID(cib_resource), task, interval_ms);
 125     lrmd__set_result(op, outcome, PCMK_EXEC_DONE, "Simulated action result");
 126     op->params = NULL;          /* TODO: Fill me in */
 127     op->t_run = (unsigned int) time(NULL);
 128     op->t_rcchange = op->t_run;
 129 
 130     op->call_id = 0;
 131     for (xop = pcmk__xe_first_child(cib_resource); xop != NULL;
 132          xop = pcmk__xe_next(xop)) {
 133 
 134         int tmp = 0;
 135 
 136         crm_element_value_int(xop, XML_LRM_ATTR_CALLID, &tmp);
 137         if (tmp > op->call_id) {
 138             op->call_id = tmp;
 139         }
 140     }
 141     op->call_id++;
 142 
 143     return op;
 144 }
 145 
 146 static xmlNode *
 147 inject_op(xmlNode * cib_resource, lrmd_event_data_t * op, int target_rc)
     /* [previous][next][first][last][top][bottom][index][help] */
 148 {
 149     return pcmk__create_history_xml(cib_resource, op, CRM_FEATURE_SET,
 150                                     target_rc, NULL, crm_system_name,
 151                                     LOG_TRACE);
 152 }
 153 
 154 static xmlNode *
 155 inject_node_state(cib_t * cib_conn, const char *node, const char *uuid)
     /* [previous][next][first][last][top][bottom][index][help] */
 156 {
 157     int rc = pcmk_ok;
 158     xmlNode *cib_object = NULL;
 159     char *xpath = crm_strdup_printf(NODE_TEMPLATE, node);
 160 
 161     if (bringing_nodes_online) {
 162         create_node_entry(cib_conn, node);
 163     }
 164 
 165     rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
 166                                cib_xpath | cib_sync_call | cib_scope_local);
 167 
 168     if (cib_object && ID(cib_object) == NULL) {
 169         crm_err("Detected multiple node_state entries for xpath=%s, bailing", xpath);
 170         crm_log_xml_warn(cib_object, "Duplicates");
 171         free(xpath);
 172         crm_exit(CRM_EX_SOFTWARE);
 173         return NULL; // not reached, but makes static analysis happy
 174     }
 175 
 176     if (rc == -ENXIO) {
 177         char *found_uuid = NULL;
 178 
 179         if (uuid == NULL) {
 180             query_node_uuid(cib_conn, node, &found_uuid, NULL);
 181         } else {
 182             found_uuid = strdup(uuid);
 183         }
 184 
 185         cib_object = create_xml_node(NULL, XML_CIB_TAG_STATE);
 186         crm_xml_add(cib_object, XML_ATTR_UUID, found_uuid);
 187         crm_xml_add(cib_object, XML_ATTR_UNAME, node);
 188         cib_conn->cmds->create(cib_conn, XML_CIB_TAG_STATUS, cib_object,
 189                                cib_sync_call | cib_scope_local);
 190         free_xml(cib_object);
 191         free(found_uuid);
 192 
 193         rc = cib_conn->cmds->query(cib_conn, xpath, &cib_object,
 194                                    cib_xpath | cib_sync_call | cib_scope_local);
 195         crm_trace("injecting node state for %s. rc is %d", node, rc);
 196     }
 197 
 198     free(xpath);
 199     CRM_ASSERT(rc == pcmk_ok);
 200     return cib_object;
 201 }
 202 
 203 static xmlNode *
 204 modify_node(cib_t * cib_conn, char *node, gboolean up)
     /* [previous][next][first][last][top][bottom][index][help] */
 205 {
 206     xmlNode *cib_node = inject_node_state(cib_conn, node, NULL);
 207 
 208     if (up) {
 209         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_YES);
 210         crm_xml_add(cib_node, XML_NODE_IS_PEER, ONLINESTATUS);
 211         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_MEMBER);
 212         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_MEMBER);
 213 
 214     } else {
 215         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
 216         crm_xml_add(cib_node, XML_NODE_IS_PEER, OFFLINESTATUS);
 217         crm_xml_add(cib_node, XML_NODE_JOIN_STATE, CRMD_JOINSTATE_DOWN);
 218         crm_xml_add(cib_node, XML_NODE_EXPECTED, CRMD_JOINSTATE_DOWN);
 219     }
 220 
 221     crm_xml_add(cib_node, XML_ATTR_ORIGIN, crm_system_name);
 222     return cib_node;
 223 }
 224 
 225 static xmlNode *
 226 find_resource_xml(xmlNode * cib_node, const char *resource)
     /* [previous][next][first][last][top][bottom][index][help] */
 227 {
 228     xmlNode *match = NULL;
 229     const char *node = crm_element_value(cib_node, XML_ATTR_UNAME);
 230     char *xpath = crm_strdup_printf(RSC_TEMPLATE, node, resource);
 231 
 232     match = get_xpath_object(xpath, cib_node, LOG_TRACE);
 233     free(xpath);
 234     return match;
 235 }
 236 
 237 
 238 static xmlNode *
 239 inject_resource(xmlNode * cib_node, const char *resource, const char *lrm_name,
     /* [previous][next][first][last][top][bottom][index][help] */
 240                 const char *rclass, const char *rtype, const char *rprovider)
 241 {
 242     xmlNode *lrm = NULL;
 243     xmlNode *container = NULL;
 244     xmlNode *cib_resource = NULL;
 245     char *xpath = NULL;
 246 
 247     cib_resource = find_resource_xml(cib_node, resource);
 248     if (cib_resource != NULL) {
 249         /* If an existing LRM history entry uses the resource name,
 250          * continue using it, even if lrm_name is different.
 251          */
 252         return cib_resource;
 253     }
 254 
 255     // Check for history entry under preferred name
 256     if (strcmp(resource, lrm_name)) {
 257         cib_resource = find_resource_xml(cib_node, lrm_name);
 258         if (cib_resource != NULL) {
 259             return cib_resource;
 260         }
 261     }
 262 
 263     /* One day, add query for class, provider, type */
 264 
 265     if (rclass == NULL || rtype == NULL) {
 266         out->err(out, "Resource %s not found in the status section of %s."
 267                  "  Please supply the class and type to continue", resource, ID(cib_node));
 268         return NULL;
 269 
 270     } else if (!pcmk__strcase_any_of(rclass, PCMK_RESOURCE_CLASS_OCF, PCMK_RESOURCE_CLASS_STONITH,
 271                                      PCMK_RESOURCE_CLASS_SERVICE, PCMK_RESOURCE_CLASS_UPSTART,
 272                                      PCMK_RESOURCE_CLASS_SYSTEMD, PCMK_RESOURCE_CLASS_LSB, NULL)) {
 273         out->err(out, "Invalid class for %s: %s", resource, rclass);
 274         return NULL;
 275 
 276     } else if (pcmk_is_set(pcmk_get_ra_caps(rclass), pcmk_ra_cap_provider)
 277                 && (rprovider == NULL)) {
 278         out->err(out, "Please specify the provider for resource %s", resource);
 279         return NULL;
 280     }
 281 
 282     xpath = (char *)xmlGetNodePath(cib_node);
 283     crm_info("Injecting new resource %s into %s '%s'", lrm_name, xpath, ID(cib_node));
 284     free(xpath);
 285 
 286     lrm = first_named_child(cib_node, XML_CIB_TAG_LRM);
 287     if (lrm == NULL) {
 288         const char *node_uuid = ID(cib_node);
 289 
 290         lrm = create_xml_node(cib_node, XML_CIB_TAG_LRM);
 291         crm_xml_add(lrm, XML_ATTR_ID, node_uuid);
 292     }
 293 
 294     container = first_named_child(lrm, XML_LRM_TAG_RESOURCES);
 295     if (container == NULL) {
 296         container = create_xml_node(lrm, XML_LRM_TAG_RESOURCES);
 297     }
 298 
 299     cib_resource = create_xml_node(container, XML_LRM_TAG_RESOURCE);
 300 
 301     // If we're creating a new entry, use the preferred name
 302     crm_xml_add(cib_resource, XML_ATTR_ID, lrm_name);
 303 
 304     crm_xml_add(cib_resource, XML_AGENT_ATTR_CLASS, rclass);
 305     crm_xml_add(cib_resource, XML_AGENT_ATTR_PROVIDER, rprovider);
 306     crm_xml_add(cib_resource, XML_ATTR_TYPE, rtype);
 307 
 308     return cib_resource;
 309 }
 310 
 311 #define XPATH_MAX 1024
 312 
 313 static int
 314 find_ticket_state(cib_t * the_cib, const char *ticket_id, xmlNode ** ticket_state_xml)
     /* [previous][next][first][last][top][bottom][index][help] */
 315 {
 316     int offset = 0;
 317     int rc = pcmk_ok;
 318     xmlNode *xml_search = NULL;
 319 
 320     char *xpath_string = NULL;
 321 
 322     CRM_ASSERT(ticket_state_xml != NULL);
 323     *ticket_state_xml = NULL;
 324 
 325     xpath_string = calloc(1, XPATH_MAX);
 326     offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "%s", "/cib/status/tickets");
 327 
 328     if (ticket_id) {
 329         offset += snprintf(xpath_string + offset, XPATH_MAX - offset, "/%s[@id=\"%s\"]",
 330                            XML_CIB_TAG_TICKET_STATE, ticket_id);
 331     }
 332     CRM_LOG_ASSERT(offset > 0);
 333     rc = the_cib->cmds->query(the_cib, xpath_string, &xml_search,
 334                               cib_sync_call | cib_scope_local | cib_xpath);
 335 
 336     if (rc != pcmk_ok) {
 337         goto bail;
 338     }
 339 
 340     crm_log_xml_debug(xml_search, "Match");
 341     if (xml_has_children(xml_search)) {
 342         if (ticket_id) {
 343             out->err(out, "Multiple ticket_states match ticket_id=%s", ticket_id);
 344         }
 345         *ticket_state_xml = xml_search;
 346     } else {
 347         *ticket_state_xml = xml_search;
 348     }
 349 
 350   bail:
 351     free(xpath_string);
 352     return rc;
 353 }
 354 
 355 static int
 356 set_ticket_state_attr(const char *ticket_id, const char *attr_name,
     /* [previous][next][first][last][top][bottom][index][help] */
 357                       const char *attr_value, cib_t * cib, int cib_options)
 358 {
 359     int rc = pcmk_ok;
 360     xmlNode *xml_top = NULL;
 361     xmlNode *ticket_state_xml = NULL;
 362 
 363     rc = find_ticket_state(cib, ticket_id, &ticket_state_xml);
 364     if (rc == pcmk_ok) {
 365         crm_debug("Found a match state for ticket: id=%s", ticket_id);
 366         xml_top = ticket_state_xml;
 367 
 368     } else if (rc != -ENXIO) {
 369         return rc;
 370 
 371     } else {
 372         xmlNode *xml_obj = NULL;
 373 
 374         xml_top = create_xml_node(NULL, XML_CIB_TAG_STATUS);
 375         xml_obj = create_xml_node(xml_top, XML_CIB_TAG_TICKETS);
 376         ticket_state_xml = create_xml_node(xml_obj, XML_CIB_TAG_TICKET_STATE);
 377         crm_xml_add(ticket_state_xml, XML_ATTR_ID, ticket_id);
 378     }
 379 
 380     crm_xml_add(ticket_state_xml, attr_name, attr_value);
 381 
 382     crm_log_xml_debug(xml_top, "Update");
 383 
 384     rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, xml_top, cib_options);
 385 
 386     free_xml(xml_top);
 387 
 388     return rc;
 389 }
 390 
 391 void
 392 modify_configuration(pe_working_set_t * data_set, cib_t *cib, pcmk_injections_t *injections)
     /* [previous][next][first][last][top][bottom][index][help] */
 393 {
 394     int rc = pcmk_ok;
 395     GList *gIter = NULL;
 396 
 397     xmlNode *cib_op = NULL;
 398     xmlNode *cib_node = NULL;
 399     xmlNode *cib_resource = NULL;
 400 
 401     lrmd_event_data_t *op = NULL;
 402 
 403     out = data_set->priv;
 404 
 405     out->message(out, "inject-modify-config", injections->quorum, injections->watchdog);
 406 
 407     if (injections->quorum) {
 408         xmlNode *top = create_xml_node(NULL, XML_TAG_CIB);
 409 
 410         /* crm_xml_add(top, XML_ATTR_DC_UUID, dc_uuid);      */
 411         crm_xml_add(top, XML_ATTR_HAVE_QUORUM, injections->quorum);
 412 
 413         rc = cib->cmds->modify(cib, NULL, top, cib_sync_call | cib_scope_local);
 414         CRM_ASSERT(rc == pcmk_ok);
 415     }
 416 
 417     if (injections->watchdog) {
 418         rc = update_attr_delegate(cib, cib_sync_call | cib_scope_local,
 419                              XML_CIB_TAG_CRMCONFIG, NULL, NULL, NULL, NULL,
 420                              XML_ATTR_HAVE_WATCHDOG, injections->watchdog, FALSE, NULL, NULL);
 421 
 422         CRM_ASSERT(rc == pcmk_ok);
 423     }
 424 
 425     for (gIter = injections->node_up; gIter != NULL; gIter = gIter->next) {
 426         char *node = (char *)gIter->data;
 427 
 428         out->message(out, "inject-modify-node", "Online", node);
 429 
 430         cib_node = modify_node(cib, node, TRUE);
 431         CRM_ASSERT(cib_node != NULL);
 432 
 433         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
 434                                       cib_sync_call | cib_scope_local);
 435         CRM_ASSERT(rc == pcmk_ok);
 436         free_xml(cib_node);
 437     }
 438 
 439     for (gIter = injections->node_down; gIter != NULL; gIter = gIter->next) {
 440         char xpath[STATUS_PATH_MAX];
 441         char *node = (char *)gIter->data;
 442 
 443         out->message(out, "inject-modify-node", "Offline", node);
 444 
 445         cib_node = modify_node(cib, node, FALSE);
 446         CRM_ASSERT(cib_node != NULL);
 447 
 448         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
 449                                       cib_sync_call | cib_scope_local);
 450         CRM_ASSERT(rc == pcmk_ok);
 451         free_xml(cib_node);
 452 
 453         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node, XML_CIB_TAG_LRM);
 454         cib->cmds->remove(cib, xpath, NULL,
 455                                       cib_xpath | cib_sync_call | cib_scope_local);
 456 
 457         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", node,
 458                  XML_TAG_TRANSIENT_NODEATTRS);
 459         cib->cmds->remove(cib, xpath, NULL,
 460                                       cib_xpath | cib_sync_call | cib_scope_local);
 461 
 462     }
 463 
 464     for (gIter = injections->node_fail; gIter != NULL; gIter = gIter->next) {
 465         char *node = (char *)gIter->data;
 466 
 467         out->message(out, "inject-modify-node", "Failing", node);
 468 
 469         cib_node = modify_node(cib, node, TRUE);
 470         crm_xml_add(cib_node, XML_NODE_IN_CLUSTER, XML_BOOLEAN_NO);
 471         CRM_ASSERT(cib_node != NULL);
 472 
 473         rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
 474                                       cib_sync_call | cib_scope_local);
 475         CRM_ASSERT(rc == pcmk_ok);
 476         free_xml(cib_node);
 477     }
 478 
 479     for (gIter = injections->ticket_grant; gIter != NULL; gIter = gIter->next) {
 480         char *ticket_id = (char *)gIter->data;
 481 
 482         out->message(out, "inject-modify-ticket", "Granting", ticket_id);
 483 
 484         rc = set_ticket_state_attr(ticket_id, "granted", "true",
 485                                    cib, cib_sync_call | cib_scope_local);
 486 
 487         CRM_ASSERT(rc == pcmk_ok);
 488     }
 489 
 490     for (gIter = injections->ticket_revoke; gIter != NULL; gIter = gIter->next) {
 491         char *ticket_id = (char *)gIter->data;
 492 
 493         out->message(out, "inject-modify-ticket", "Revoking", ticket_id);
 494 
 495         rc = set_ticket_state_attr(ticket_id, "granted", "false",
 496                                    cib, cib_sync_call | cib_scope_local);
 497 
 498         CRM_ASSERT(rc == pcmk_ok);
 499     }
 500 
 501     for (gIter = injections->ticket_standby; gIter != NULL; gIter = gIter->next) {
 502         char *ticket_id = (char *)gIter->data;
 503 
 504         out->message(out, "inject-modify-ticket", "Standby", ticket_id);
 505 
 506         rc = set_ticket_state_attr(ticket_id, "standby", "true",
 507                                    cib, cib_sync_call | cib_scope_local);
 508 
 509         CRM_ASSERT(rc == pcmk_ok);
 510     }
 511 
 512     for (gIter = injections->ticket_activate; gIter != NULL; gIter = gIter->next) {
 513         char *ticket_id = (char *)gIter->data;
 514 
 515         out->message(out, "inject-modify-ticket", "Activating", ticket_id);
 516 
 517         rc = set_ticket_state_attr(ticket_id, "standby", "false",
 518                                    cib, cib_sync_call | cib_scope_local);
 519 
 520         CRM_ASSERT(rc == pcmk_ok);
 521     }
 522 
 523     for (gIter = injections->op_inject; gIter != NULL; gIter = gIter->next) {
 524         char *spec = (char *)gIter->data;
 525 
 526         int rc = 0;
 527         int outcome = PCMK_OCF_OK;
 528         guint interval_ms = 0;
 529 
 530         char *key = NULL;
 531         char *node = NULL;
 532         char *task = NULL;
 533         char *resource = NULL;
 534 
 535         const char *rtype = NULL;
 536         const char *rclass = NULL;
 537         const char *rprovider = NULL;
 538 
 539         pe_resource_t *rsc = NULL;
 540 
 541         out->message(out, "inject-spec", spec);
 542 
 543         key = calloc(1, strlen(spec) + 1);
 544         node = calloc(1, strlen(spec) + 1);
 545         rc = sscanf(spec, "%[^@]@%[^=]=%d", key, node, &outcome);
 546         if (rc != 3) {
 547             out->err(out, "Invalid operation spec: %s.  Only found %d fields", spec, rc);
 548             free(key);
 549             free(node);
 550             continue;
 551         }
 552 
 553         parse_op_key(key, &resource, &task, &interval_ms);
 554 
 555         rsc = pe_find_resource(data_set->resources, resource);
 556         if (rsc == NULL) {
 557             out->err(out, "Invalid resource name: %s", resource);
 558         } else {
 559             rclass = crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS);
 560             rtype = crm_element_value(rsc->xml, XML_ATTR_TYPE);
 561             rprovider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER);
 562 
 563             cib_node = inject_node_state(cib, node, NULL);
 564             CRM_ASSERT(cib_node != NULL);
 565 
 566             update_failcounts(cib_node, resource, task, interval_ms, outcome);
 567 
 568             cib_resource = inject_resource(cib_node, resource, resource,
 569                                            rclass, rtype, rprovider);
 570             CRM_ASSERT(cib_resource != NULL);
 571 
 572             op = create_op(cib_resource, task, interval_ms, outcome);
 573             CRM_ASSERT(op != NULL);
 574 
 575             cib_op = inject_op(cib_resource, op, 0);
 576             CRM_ASSERT(cib_op != NULL);
 577             lrmd_free_event(op);
 578 
 579             rc = cib->cmds->modify(cib, XML_CIB_TAG_STATUS, cib_node,
 580                                           cib_sync_call | cib_scope_local);
 581             CRM_ASSERT(rc == pcmk_ok);
 582         }
 583         free(task);
 584         free(node);
 585         free(key);
 586     }
 587 
 588     if (!out->is_quiet(out)) {
 589         out->end_list(out);
 590     }
 591 }
 592 
 593 static gboolean
 594 exec_pseudo_action(crm_graph_t * graph, crm_action_t * action)
     /* [previous][next][first][last][top][bottom][index][help] */
 595 {
 596     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 597     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
 598 
 599     crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 600     out->message(out, "inject-pseudo-action", node, task);
 601 
 602     pcmk__update_graph(graph, action);
 603     return TRUE;
 604 }
 605 
 606 static gboolean
 607 exec_rsc_action(crm_graph_t * graph, crm_action_t * action)
     /* [previous][next][first][last][top][bottom][index][help] */
 608 {
 609     int rc = 0;
 610     GList *gIter = NULL;
 611     lrmd_event_data_t *op = NULL;
 612     int target_outcome = PCMK_OCF_OK;
 613 
 614     const char *rtype = NULL;
 615     const char *rclass = NULL;
 616     const char *resource = NULL;
 617     const char *rprovider = NULL;
 618     const char *lrm_name = NULL;
 619     const char *operation = crm_element_value(action->xml, "operation");
 620     const char *target_rc_s = crm_meta_value(action->params, XML_ATTR_TE_TARGET_RC);
 621 
 622     xmlNode *cib_node = NULL;
 623     xmlNode *cib_resource = NULL;
 624     xmlNode *action_rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 625 
 626     char *node = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 627     char *uuid = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET_UUID);
 628     const char *router_node = crm_element_value(action->xml, XML_LRM_ATTR_ROUTER_NODE);
 629 
 630     if (pcmk__strcase_any_of(operation, CRM_OP_PROBED, CRM_OP_REPROBE, NULL)) {
 631         crm_info("Skipping %s op for %s", operation, node);
 632         goto done;
 633     }
 634 
 635     if (action_rsc == NULL) {
 636         crm_log_xml_err(action->xml, "Bad");
 637         free(node); free(uuid);
 638         return FALSE;
 639     }
 640 
 641     /* Look for the preferred name
 642      * If not found, try the expected 'local' name
 643      * If not found use the preferred name anyway
 644      */
 645     resource = crm_element_value(action_rsc, XML_ATTR_ID);
 646     CRM_ASSERT(resource != NULL); // makes static analysis happy
 647     lrm_name = resource; // Preferred name when writing history
 648     if (pe_find_resource(fake_resource_list, resource) == NULL) {
 649         const char *longname = crm_element_value(action_rsc, XML_ATTR_ID_LONG);
 650 
 651         if (longname && pe_find_resource(fake_resource_list, longname)) {
 652             resource = longname;
 653         }
 654     }
 655 
 656     if (pcmk__strcase_any_of(operation, "delete", RSC_METADATA, NULL)) {
 657         out->message(out, "inject-rsc-action", resource, operation, node, (guint) 0);
 658         goto done;
 659     }
 660 
 661     rclass = crm_element_value(action_rsc, XML_AGENT_ATTR_CLASS);
 662     rtype = crm_element_value(action_rsc, XML_ATTR_TYPE);
 663     rprovider = crm_element_value(action_rsc, XML_AGENT_ATTR_PROVIDER);
 664 
 665     pcmk__scan_min_int(target_rc_s, &target_outcome, 0);
 666 
 667     CRM_ASSERT(fake_cib->cmds->query(fake_cib, NULL, NULL, cib_sync_call | cib_scope_local) ==
 668                pcmk_ok);
 669 
 670     cib_node = inject_node_state(fake_cib, node, (router_node? node : uuid));
 671     CRM_ASSERT(cib_node != NULL);
 672 
 673     cib_resource = inject_resource(cib_node, resource, lrm_name,
 674                                    rclass, rtype, rprovider);
 675     if (cib_resource == NULL) {
 676         crm_err("invalid resource in transition");
 677         free(node); free(uuid);
 678         free_xml(cib_node);
 679         return FALSE;
 680     }
 681 
 682     op = pcmk__event_from_graph_action(cib_resource, action, PCMK_EXEC_DONE,
 683                                        target_outcome, "User-injected result");
 684 
 685     out->message(out, "inject-rsc-action", resource, op->op_type, node, op->interval_ms);
 686 
 687     for (gIter = fake_op_fail_list; gIter != NULL; gIter = gIter->next) {
 688         char *spec = (char *)gIter->data;
 689         char *key = NULL;
 690         const char *match_name = NULL;
 691 
 692         // Allow user to specify anonymous clone with or without instance number
 693         key = crm_strdup_printf(PCMK__OP_FMT "@%s=", resource, op->op_type,
 694                                 op->interval_ms, node);
 695         if (strncasecmp(key, spec, strlen(key)) == 0) {
 696             match_name = resource;
 697         }
 698         free(key);
 699 
 700         if ((match_name == NULL) && strcmp(resource, lrm_name)) {
 701             key = crm_strdup_printf(PCMK__OP_FMT "@%s=", lrm_name, op->op_type,
 702                                     op->interval_ms, node);
 703             if (strncasecmp(key, spec, strlen(key)) == 0) {
 704                 match_name = lrm_name;
 705             }
 706             free(key);
 707         }
 708 
 709         if (match_name != NULL) {
 710 
 711             rc = sscanf(spec, "%*[^=]=%d", (int *) &op->rc);
 712             // ${match_name}_${task}_${interval_in_ms}@${node}=${rc}
 713 
 714             if (rc != 1) {
 715                 out->err(out,
 716                          "Invalid failed operation spec: %s. Result code must be integer",
 717                          spec);
 718                 continue;
 719             }
 720             crm__set_graph_action_flags(action, pcmk__graph_action_failed);
 721             graph->abort_priority = INFINITY;
 722             out->info(out, "Pretending action %d failed with rc=%d", action->id, op->rc);
 723             update_failcounts(cib_node, match_name, op->op_type,
 724                               op->interval_ms, op->rc);
 725             break;
 726         }
 727     }
 728 
 729     inject_op(cib_resource, op, target_outcome);
 730     lrmd_free_event(op);
 731 
 732     rc = fake_cib->cmds->modify(fake_cib, XML_CIB_TAG_STATUS, cib_node,
 733                                   cib_sync_call | cib_scope_local);
 734     CRM_ASSERT(rc == pcmk_ok);
 735 
 736   done:
 737     free(node); free(uuid);
 738     free_xml(cib_node);
 739     crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 740     pcmk__update_graph(graph, action);
 741     return TRUE;
 742 }
 743 
 744 static gboolean
 745 exec_crmd_action(crm_graph_t * graph, crm_action_t * action)
     /* [previous][next][first][last][top][bottom][index][help] */
 746 {
 747     const char *node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
 748     const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
 749     xmlNode *rsc = first_named_child(action->xml, XML_CIB_TAG_RESOURCE);
 750 
 751     crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 752     out->message(out, "inject-cluster-action", node, task, rsc);
 753     pcmk__update_graph(graph, action);
 754     return TRUE;
 755 }
 756 
 757 static gboolean
 758 exec_stonith_action(crm_graph_t * graph, crm_action_t * action)
     /* [previous][next][first][last][top][bottom][index][help] */
 759 {
 760     const char *op = crm_meta_value(action->params, "stonith_action");
 761     char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
 762 
 763     out->message(out, "inject-fencing-action", target, op);
 764 
 765     if(!pcmk__str_eq(op, "on", pcmk__str_casei)) {
 766         int rc = 0;
 767         char xpath[STATUS_PATH_MAX];
 768         xmlNode *cib_node = modify_node(fake_cib, target, FALSE);
 769 
 770         crm_xml_add(cib_node, XML_ATTR_ORIGIN, __func__);
 771         CRM_ASSERT(cib_node != NULL);
 772 
 773         rc = fake_cib->cmds->replace(fake_cib, XML_CIB_TAG_STATUS, cib_node,
 774                                    cib_sync_call | cib_scope_local);
 775         CRM_ASSERT(rc == pcmk_ok);
 776 
 777         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM);
 778         fake_cib->cmds->remove(fake_cib, xpath, NULL,
 779                                       cib_xpath | cib_sync_call | cib_scope_local);
 780 
 781         snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target,
 782                  XML_TAG_TRANSIENT_NODEATTRS);
 783         fake_cib->cmds->remove(fake_cib, xpath, NULL,
 784                                       cib_xpath | cib_sync_call | cib_scope_local);
 785 
 786         free_xml(cib_node);
 787     }
 788 
 789     crm__set_graph_action_flags(action, pcmk__graph_action_confirmed);
 790     pcmk__update_graph(graph, action);
 791     free(target);
 792     return TRUE;
 793 }
 794 
 795 enum transition_status
 796 run_simulation(pe_working_set_t * data_set, cib_t *cib, GList *op_fail_list)
     /* [previous][next][first][last][top][bottom][index][help] */
 797 {
 798     crm_graph_t *transition = NULL;
 799     enum transition_status graph_rc;
 800 
 801     crm_graph_functions_t exec_fns = {
 802         exec_pseudo_action,
 803         exec_rsc_action,
 804         exec_crmd_action,
 805         exec_stonith_action,
 806     };
 807 
 808     out = data_set->priv;
 809 
 810     fake_cib = cib;
 811     fake_op_fail_list = op_fail_list;
 812 
 813     if (!out->is_quiet(out)) {
 814         out->begin_list(out, NULL, NULL, "Executing Cluster Transition");
 815     }
 816 
 817     pcmk__set_graph_functions(&exec_fns);
 818     transition = pcmk__unpack_graph(data_set->graph, crm_system_name);
 819     pcmk__log_graph(LOG_DEBUG, transition);
 820 
 821     fake_resource_list = data_set->resources;
 822     do {
 823         graph_rc = pcmk__execute_graph(transition);
 824 
 825     } while (graph_rc == transition_active);
 826     fake_resource_list = NULL;
 827 
 828     if (graph_rc != transition_complete) {
 829         out->err(out, "Transition failed: %s",
 830                  pcmk__graph_status2text(graph_rc));
 831         pcmk__log_graph(LOG_ERR, transition);
 832     }
 833     pcmk__free_graph(transition);
 834     if (graph_rc != transition_complete) {
 835         out->err(out, "An invalid transition was produced");
 836     }
 837 
 838     if (!out->is_quiet(out)) {
 839         xmlNode *cib_object = NULL;
 840         int rc = fake_cib->cmds->query(fake_cib, NULL, &cib_object, cib_sync_call | cib_scope_local);
 841 
 842         CRM_ASSERT(rc == pcmk_ok);
 843         pe_reset_working_set(data_set);
 844         data_set->input = cib_object;
 845 
 846         out->end_list(out);
 847     }
 848 
 849     return graph_rc;
 850 }

/* [previous][next][first][last][top][bottom][index][help] */