root/tools/crm_resource_runtime.c

/* */

DEFINITIONS

1. build_node_info_list
2. cli_resource_search
3. find_resource_attr
4. find_matching_attr_resources_recursive
5. find_matching_attr_resources
6. update_element_attribute
7. resources_with_attr
8. free_attr_update_data
9. update_attribute
10. cli_resource_update_attribute
11. cli_resource_delete_attribute
12. send_lrm_rsc_op
13. rsc_fail_name
14. clear_rsc_history
15. clear_rsc_failures
16. clear_rsc_fail_attrs
17. cli_resource_delete
18. cli_cleanup_all
19. check_role
20. check_managed
21. check_locked
22. node_is_unhealthy
23. check_node_health
24. cli_resource_check
25. cli_resource_fail
26. generate_resource_params
27. resource_is_running_on
28. get_active_resources
29. dump_list
30. display_list
31. update_scheduler_input
32. update_scheduler_input_to_cib
33. update_dataset
34. max_rsc_stop_timeout
35. wait_time_estimate
36. cli_resource_restart
37. action_is_pending
38. actions_are_pending
39. print_pending_actions
40. wait_till_stable
41. get_action
42. set_agent_environment
43. apply_overrides
44. cli_resource_execute_from_params
45. get_action_timeout
46. cli_resource_execute
47. cli_resource_move

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <limits.h>
  14 #include <glib.h>
  15 #include <libxml/tree.h>
  16 
  17 #include <crm/common/ipc_attrd_internal.h>
  18 #include <crm/common/ipc_controld.h>
  19 #include <crm/common/lists_internal.h>
  20 #include <crm/services_internal.h>
  21 
  22 #include <crm_resource.h>
  23 
  24 static GList *
  25 build_node_info_list(const pcmk_resource_t *rsc)
     /*  */
  26 {
  27     GList *retval = NULL;
  28 
  29     for (const GList *iter = rsc->children; iter != NULL; iter = iter->next) {
  30         const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
  31 
  32         for (const GList *iter2 = child->running_on;
  33              iter2 != NULL; iter2 = iter2->next) {
  34 
  35             const pcmk_node_t *node = (const pcmk_node_t *) iter2->data;
  36             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
  37 
  38             ni->node_name = node->details->uname;
  39             ni->promoted = pcmk_is_set(rsc->flags, pcmk_rsc_promotable) &&
  40                            child->fns->state(child, TRUE) == pcmk_role_promoted;
  41 
  42             retval = g_list_prepend(retval, ni);
  43         }
  44     }
  45 
  46     return retval;
  47 }
  48 
  49 GList *
  50 cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
  51                     pcmk_scheduler_t *scheduler)
  52 {
  53     GList *retval = NULL;
  54     const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
  55 
  56     if (pcmk__is_clone(rsc)) {
  57         retval = build_node_info_list(rsc);
  58 
  59     /* The anonymous clone children's common ID is supplied */
  60     } else if (pcmk__is_clone(parent)
  61                && !pcmk_is_set(rsc->flags, pcmk_rsc_unique)
  62                && rsc->clone_name
  63                && pcmk__str_eq(requested_name, rsc->clone_name, pcmk__str_none)
  64                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_none)) {
  65 
  66         retval = build_node_info_list(parent);
  67 
  68     } else if (rsc->running_on != NULL) {
  69         for (GList *iter = rsc->running_on; iter != NULL; iter = iter->next) {
  70             pcmk_node_t *node = (pcmk_node_t *) iter->data;
  71             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
  72 
  73             ni->node_name = node->details->uname;
  74             ni->promoted = (rsc->fns->state(rsc, TRUE) == pcmk_role_promoted);
  75 
  76             retval = g_list_prepend(retval, ni);
  77         }
  78     }
  79 
  80     return retval;
  81 }
  82 
  83 // \return Standard Pacemaker return code
  84 static int
  85 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
     /*  */
  86                    const char *rsc, const char *attr_set_type, const char *set_name,
  87                    const char *attr_id, const char *attr_name, xmlNode **result)
  88 {
  89     xmlNode *xml_search;
  90     int rc = pcmk_rc_ok;
  91     GString *xpath = NULL;
  92     const char *xpath_base = NULL;
  93 
  94     if (result) {
  95         *result = NULL;
  96     }
  97 
  98     if(the_cib == NULL) {
  99         return ENOTCONN;
 100     }
 101 
 102     xpath_base = pcmk_cib_xpath_for(PCMK_XE_RESOURCES);
 103     if (xpath_base == NULL) {
 104         crm_err(PCMK_XE_RESOURCES " CIB element not known (bug?)");
 105         return ENOMSG;
 106     }
 107 
 108     xpath = g_string_sized_new(1024);
 109     pcmk__g_strcat(xpath,
 110                    xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);
 111 
 112     if (attr_set_type != NULL) {
 113         pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
 114         if (set_name != NULL) {
 115             pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
 116                            NULL);
 117         }
 118     }
 119 
 120     g_string_append(xpath, "//" PCMK_XE_NVPAIR);
 121 
 122     if (attr_id != NULL && attr_name!= NULL) {
 123         pcmk__g_strcat(xpath,
 124                        "[@" PCMK_XA_ID "='", attr_id, "' "
 125                        "and @" PCMK_XA_NAME "='", attr_name, "']", NULL);
 126 
 127     } else if (attr_id != NULL) {
 128         pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);
 129 
 130     } else if (attr_name != NULL) {
 131         pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
 132     }
 133 
 134     rc = the_cib->cmds->query(the_cib, (const char *) xpath->str, &xml_search,
 135                               cib_sync_call | cib_scope_local | cib_xpath);
 136     rc = pcmk_legacy2rc(rc);
 137 
 138     if (rc == pcmk_rc_ok) {
 139         crm_log_xml_debug(xml_search, "Match");
 140         if (xml_search->children != NULL) {
 141             rc = ENOTUNIQ;
 142             pcmk__warn_multiple_name_matches(out, xml_search, attr_name);
 143             out->spacer(out);
 144         }
 145     }
 146 
 147     if (result) {
 148         *result = xml_search;
 149     } else {
 150         free_xml(xml_search);
 151     }
 152 
 153     g_string_free(xpath, TRUE);
 154     return rc;
 155 }
 156 
 157 /* PRIVATE. Use the find_matching_attr_resources instead. */
 158 static void
 159 find_matching_attr_resources_recursive(pcmk__output_t *out,
     /*  */
 160                                        GList /* <pcmk_resource_t*> */ **result,
 161                                        pcmk_resource_t *rsc, const char * attr_set,
 162                                        const char * attr_set_type, const char * attr_id,
 163                                        const char * attr_name, cib_t * cib, int depth)
 164 {
 165     int rc = pcmk_rc_ok;
 166     char *lookup_id = clone_strip(rsc->id);
 167 
 168     /* visit the children */
 169     for(GList *gIter = rsc->children; gIter; gIter = gIter->next) {
 170         find_matching_attr_resources_recursive(out, result,
 171                                                (pcmk_resource_t *) gIter->data,
 172                                                attr_set, attr_set_type, attr_id,
 173                                                attr_name, cib, depth+1);
 174         /* do it only once for clones */
 175         if (pcmk__is_clone(rsc)) {
 176             break;
 177         }
 178     }
 179 
 180     rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 181                             attr_set, attr_id, attr_name, NULL);
 182     /* Post-order traversal.
 183      * The root is always on the list and it is the last item. */
 184     if((0 == depth) || (pcmk_rc_ok == rc)) {
 185         /* push the head */
 186         *result = g_list_append(*result, rsc);
 187     }
 188 
 189     free(lookup_id);
 190 }
 191 
 192 
 193 /* The result is a linearized pre-ordered tree of resources. */
 194 static GList/*<pcmk_resource_t*>*/ *
 195 find_matching_attr_resources(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
 196                              const char * rsc_id, const char * attr_set,
 197                              const char * attr_set_type, const char * attr_id,
 198                              const char * attr_name, cib_t * cib, const char * cmd,
 199                              gboolean force)
 200 {
 201     int rc = pcmk_rc_ok;
 202     char *lookup_id = NULL;
 203     GList * result = NULL;
 204     /* If --force is used, update only the requested resource (clone or primitive).
 205      * Otherwise, if the primitive has the attribute, use that.
 206      * Otherwise use the clone. */
 207     if(force == TRUE) {
 208         return g_list_append(result, rsc);
 209     }
 210     if (pcmk__is_clone(rsc->parent)) {
 211         int rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
 212                                     attr_set, attr_id, attr_name, NULL);
 213 
 214         if(rc != pcmk_rc_ok) {
 215             rsc = rsc->parent;
 216             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
 217                       cmd, attr_name, rsc->id, rsc_id);
 218         }
 219         return g_list_append(result, rsc);
 220 
 221     } else if ((rsc->parent == NULL) && (rsc->children != NULL)
 222                && pcmk__is_clone(rsc)) {
 223         pcmk_resource_t *child = rsc->children->data;
 224 
 225         if (pcmk__is_primitive(child)) {
 226             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
 227             rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
 228                                     attr_set_type, attr_set, attr_id, attr_name, NULL);
 229 
 230             if(rc == pcmk_rc_ok) {
 231                 rsc = child;
 232                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
 233                           attr_name, lookup_id, cmd, rsc_id);
 234             }
 235 
 236             free(lookup_id);
 237         }
 238         return g_list_append(result, rsc);
 239     }
 240     /* If the resource is a group ==> children inherit the attribute if defined. */
 241     find_matching_attr_resources_recursive(out, &result, rsc, attr_set,
 242                                            attr_set_type, attr_id, attr_name,
 243                                            cib, 0);
 244     return result;
 245 }
 246 
 247 static int
 248 update_element_attribute(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
 249                          cib_t *cib, const char *attr_name, const char *attr_value)
 250 {
 251     int rc = pcmk_rc_ok;
 252 
 253     if (cib == NULL) {
 254         return ENOTCONN;
 255     }
 256 
 257     crm_xml_add(rsc->xml, attr_name, attr_value);
 258 
 259     rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->xml, cib_sync_call);
 260     rc = pcmk_legacy2rc(rc);
 261     if (rc == pcmk_rc_ok) {
 262         out->info(out, "Set attribute: " PCMK_XA_NAME "=%s value=%s",
 263                   attr_name, attr_value);
 264     }
 265 
 266     return rc;
 267 }
 268 
 269 static int
 270 resources_with_attr(pcmk__output_t *out, cib_t *cib, pcmk_resource_t *rsc,
     /*  */
 271                     const char *requested_name, const char *attr_set,
 272                     const char *attr_set_type, const char *attr_id,
 273                     const char *attr_name, const char *top_id, gboolean force,
 274                     GList **resources)
 275 {
 276     if (pcmk__str_eq(attr_set_type, PCMK_XE_INSTANCE_ATTRIBUTES,
 277                      pcmk__str_casei)) {
 278         if (!force) {
 279             xmlNode *xml_search = NULL;
 280             int rc = pcmk_rc_ok;
 281 
 282             rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
 283                                     PCMK_XE_META_ATTRIBUTES, attr_set, attr_id,
 284                                     attr_name, &xml_search);
 285 
 286             if (rc == pcmk_rc_ok || rc == ENOTUNIQ) {
 287                 char *found_attr_id = NULL;
 288 
 289                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 290 
 291                 if (!out->is_quiet(out)) {
 292                     out->err(out,
 293                              "WARNING: There is already a meta attribute "
 294                              "for '%s' called '%s' (id=%s)",
 295                              top_id, attr_name, found_attr_id);
 296                     out->err(out,
 297                              "         Delete '%s' first or use the force option "
 298                              "to override", found_attr_id);
 299                 }
 300 
 301                 free(found_attr_id);
 302                 free_xml(xml_search);
 303                 return ENOTUNIQ;
 304             }
 305 
 306             free_xml(xml_search);
 307         }
 308 
 309         *resources = g_list_append(*resources, rsc);
 310 
 311     } else {
 312         *resources = find_matching_attr_resources(out, rsc, requested_name,
 313                                                   attr_set, attr_set_type,
 314                                                   attr_id, attr_name, cib,
 315                                                   "update", force);
 316     }
 317 
 318     /* If the user specified attr_set or attr_id, the intent is to modify a
 319      * single resource, which will be the last item in the list.
 320      */
 321     if ((attr_set != NULL) || (attr_id != NULL)) {
 322         GList *last = g_list_last(*resources);
 323 
 324         *resources = g_list_remove_link(*resources, last);
 325         g_list_free(*resources);
 326         *resources = last;
 327     }
 328 
 329     return pcmk_rc_ok;
 330 }
 331 
 332 static void
 333 free_attr_update_data(gpointer data)
     /*  */
 334 {
 335     attr_update_data_t *ud = data;
 336 
 337     if (ud == NULL) {
 338         return;
 339     }
 340 
 341     free(ud->attr_set_type);
 342     free(ud->attr_set_id);
 343     free(ud->attr_name);
 344     free(ud->attr_value);
 345     free(ud->given_rsc_id);
 346     free(ud->found_attr_id);
 347     free(ud);
 348 }
 349 
 350 static int
 351 update_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 352                  const char *attr_set, const char *attr_set_type,
 353                  const char *attr_id, const char *attr_name,
 354                  const char *attr_value, gboolean recursive, cib_t *cib,
 355                  gboolean force, GList **results)
 356 {
 357     pcmk__output_t *out = rsc->cluster->priv;
 358     int rc = pcmk_rc_ok;
 359 
 360     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 361     const char *top_id = pe__const_top_resource(rsc, false)->id;
 362 
 363     if ((attr_id == NULL) && !force) {
 364         find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
 365                            attr_name, NULL);
 366     }
 367 
 368     rc = resources_with_attr(out, cib, rsc, requested_name, attr_set, attr_set_type,
 369                              attr_id, attr_name, top_id, force, &resources);
 370 
 371     if (rc != pcmk_rc_ok) {
 372         return rc;
 373     }
 374 
 375     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 376         char *lookup_id = NULL;
 377         char *local_attr_set = NULL;
 378         char *found_attr_id = NULL;
 379         const char *rsc_attr_id = attr_id;
 380         const char *rsc_attr_set = attr_set;
 381 
 382         xmlNode *xml_top = NULL;
 383         xmlNode *xml_obj = NULL;
 384         xmlNode *xml_search = NULL;
 385 
 386         rsc = (pcmk_resource_t *) iter->data;
 387 
 388         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
 389         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 390                                 attr_set, attr_id, attr_name, &xml_search);
 391 
 392         switch (rc) {
 393             case pcmk_rc_ok:
 394                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 395                 crm_debug("Found a match for " PCMK_XA_NAME "='%s': "
 396                           PCMK_XA_ID "='%s'", attr_name, found_attr_id);
 397                 rsc_attr_id = found_attr_id;
 398                 break;
 399 
 400             case ENXIO:
 401                 if (rsc_attr_set == NULL) {
 402                     local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
 403                                                        attr_set_type);
 404                     rsc_attr_set = local_attr_set;
 405                 }
 406                 if (rsc_attr_id == NULL) {
 407                     found_attr_id = crm_strdup_printf("%s-%s",
 408                                                       rsc_attr_set, attr_name);
 409                     rsc_attr_id = found_attr_id;
 410                 }
 411 
 412                 xml_top = pcmk__xe_create(NULL, (const char *) rsc->xml->name);
 413                 crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);
 414 
 415                 xml_obj = pcmk__xe_create(xml_top, attr_set_type);
 416                 crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
 417                 break;
 418 
 419             default:
 420                 free(lookup_id);
 421                 free(found_attr_id);
 422                 free_xml(xml_search);
 423                 g_list_free(resources);
 424                 return rc;
 425         }
 426 
 427         xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
 428                                         attr_value);
 429         if (xml_top == NULL) {
 430             xml_top = xml_obj;
 431         }
 432 
 433         crm_log_xml_debug(xml_top, "Update");
 434 
 435         rc = cib->cmds->modify(cib, PCMK_XE_RESOURCES, xml_top, cib_sync_call);
 436         rc = pcmk_legacy2rc(rc);
 437         if (rc == pcmk_rc_ok) {
 438             attr_update_data_t *ud = pcmk__assert_alloc(1, sizeof(attr_update_data_t));
 439 
 440             if (attr_set_type == NULL) {
 441                 attr_set_type = (const char *) xml_search->parent->name;
 442             }
 443 
 444             if (rsc_attr_set == NULL) {
 445                 rsc_attr_set = crm_element_value(xml_search->parent, PCMK_XA_ID);
 446             }
 447 
 448             ud->attr_set_type = pcmk__str_copy(attr_set_type);
 449             ud->attr_set_id = pcmk__str_copy(rsc_attr_set);
 450             ud->attr_name = pcmk__str_copy(attr_name);
 451             ud->attr_value = pcmk__str_copy(attr_value);
 452             ud->given_rsc_id = pcmk__str_copy(lookup_id);
 453             ud->found_attr_id = pcmk__str_copy(found_attr_id);
 454             ud->rsc = rsc;
 455 
 456             *results = g_list_append(*results, ud);
 457         }
 458 
 459         free_xml(xml_top);
 460         free_xml(xml_search);
 461 
 462         free(lookup_id);
 463         free(found_attr_id);
 464         free(local_attr_set);
 465 
 466         if (recursive
 467             && pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES,
 468                             pcmk__str_casei)) {
 469             /* We want to set the attribute only on resources explicitly
 470              * colocated with this one, so we use rsc->rsc_cons_lhs directly
 471              * rather than the with_this_colocations() method.
 472              */
 473             pcmk__set_rsc_flags(rsc, pcmk_rsc_detect_loop);
 474             for (GList *lpc = rsc->rsc_cons_lhs; lpc != NULL; lpc = lpc->next) {
 475                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
 476 
 477                 crm_debug("Checking %s %d", cons->id, cons->score);
 478 
 479                 if (pcmk_is_set(cons->dependent->flags, pcmk_rsc_detect_loop)
 480                     || (cons->score <= 0)) {
 481                     continue;
 482                 }
 483 
 484                 crm_debug("Setting %s=%s for dependent resource %s",
 485                           attr_name, attr_value, cons->dependent->id);
 486                 update_attribute(cons->dependent, cons->dependent->id, NULL,
 487                                  attr_set_type, NULL, attr_name, attr_value,
 488                                  recursive, cib, force, results);
 489             }
 490         }
 491     }
 492 
 493     g_list_free(resources);
 494     return rc;
 495 }
 496 
 497 // \return Standard Pacemaker return code
 498 int
 499 cli_resource_update_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 500                               const char *attr_set, const char *attr_set_type,
 501                               const char *attr_id, const char *attr_name,
 502                               const char *attr_value, gboolean recursive,
 503                               cib_t *cib, gboolean force)
 504 {
 505     static bool need_init = true;
 506     int rc = pcmk_rc_ok;
 507 
 508     GList *results = NULL;
 509     pcmk__output_t *out = rsc->cluster->priv;
 510 
 511     /* If we were asked to update the attribute in a resource element (for
 512      * instance, <primitive class="ocf">) there's really not much we need to do.
 513      */
 514     if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 515         return update_element_attribute(out, rsc, cib, attr_name, attr_value);
 516     }
 517 
 518     /* One time initialization - clear flags so we can detect loops */
 519     if (need_init) {
 520         need_init = false;
 521         pcmk__unpack_constraints(rsc->cluster);
 522         pe__clear_resource_flags_on_all(rsc->cluster, pcmk_rsc_detect_loop);
 523     }
 524 
 525     rc = update_attribute(rsc, requested_name, attr_set, attr_set_type,
 526                           attr_id, attr_name, attr_value, recursive, cib,
 527                           force, &results);
 528 
 529     if (rc == pcmk_rc_ok) {
 530         if (results == NULL) {
 531             return rc;
 532         }
 533 
 534         out->message(out, "attribute-changed-list", results);
 535         g_list_free_full(results, free_attr_update_data);
 536     }
 537 
 538     return rc;
 539 }
 540 
 541 // \return Standard Pacemaker return code
 542 int
 543 cli_resource_delete_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 544                               const char *attr_set, const char *attr_set_type,
 545                               const char *attr_id, const char *attr_name,
 546                               cib_t *cib, int cib_options, gboolean force)
 547 {
 548     pcmk__output_t *out = rsc->cluster->priv;
 549     int rc = pcmk_rc_ok;
 550     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 551 
 552     if ((attr_id == NULL) && !force) {
 553         find_resource_attr(out, cib, PCMK_XA_ID,
 554                            pe__const_top_resource(rsc, false)->id, NULL,
 555                            NULL, NULL, attr_name, NULL);
 556     }
 557 
 558     if (pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES, pcmk__str_casei)) {
 559         resources = find_matching_attr_resources(out, rsc, requested_name,
 560                                                  attr_set, attr_set_type,
 561                                                  attr_id, attr_name, cib,
 562                                                  "delete", force);
 563 
 564     } else if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 565         pcmk__xe_remove_attr(rsc->xml, attr_name);
 566         pcmk__assert(cib != NULL);
 567         rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc->xml, cib_options);
 568         rc = pcmk_legacy2rc(rc);
 569         if (rc == pcmk_rc_ok) {
 570             out->info(out, "Deleted attribute: %s", attr_name);
 571         }
 572         return rc;
 573 
 574     } else {
 575         resources = g_list_append(resources, rsc);
 576     }
 577 
 578     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 579         char *lookup_id = NULL;
 580         xmlNode *xml_obj = NULL;
 581         xmlNode *xml_search = NULL;
 582         char *found_attr_id = NULL;
 583         const char *rsc_attr_id = attr_id;
 584 
 585         rsc = (pcmk_resource_t *) iter->data;
 586 
 587         lookup_id = clone_strip(rsc->id);
 588         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 589                                 attr_set, attr_id, attr_name, &xml_search);
 590         switch (rc) {
 591             case pcmk_rc_ok:
 592                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 593                 free_xml(xml_search);
 594                 break;
 595 
 596             case ENXIO:
 597                 free(lookup_id);
 598                 free_xml(xml_search);
 599                 continue;
 600 
 601             default:
 602                 free(lookup_id);
 603                 free_xml(xml_search);
 604                 g_list_free(resources);
 605                 return rc;
 606         }
 607 
 608         if (rsc_attr_id == NULL) {
 609             rsc_attr_id = found_attr_id;
 610         }
 611 
 612         xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
 613         crm_log_xml_debug(xml_obj, "Delete");
 614 
 615         pcmk__assert(cib != NULL);
 616         rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, xml_obj, cib_options);
 617         rc = pcmk_legacy2rc(rc);
 618 
 619         if (rc == pcmk_rc_ok) {
 620             out->info(out, "Deleted '%s' option: " PCMK_XA_ID "=%s%s%s%s%s",
 621                       lookup_id, found_attr_id,
 622                       ((attr_set == NULL)? "" : " set="),
 623                       pcmk__s(attr_set, ""),
 624                       ((attr_name == NULL)? "" : " " PCMK_XA_NAME "="),
 625                       pcmk__s(attr_name, ""));
 626         }
 627 
 628         free(lookup_id);
 629         free_xml(xml_obj);
 630         free(found_attr_id);
 631     }
 632     g_list_free(resources);
 633     return rc;
 634 }
 635 
 636 // \return Standard Pacemaker return code
 637 static int
 638 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
     /*  */
 639                 const char *host_uname, const char *rsc_id,
 640                 pcmk_scheduler_t *scheduler)
 641 {
 642     pcmk__output_t *out = scheduler->priv;
 643     const char *router_node = host_uname;
 644     const char *rsc_api_id = NULL;
 645     const char *rsc_long_id = NULL;
 646     const char *rsc_class = NULL;
 647     const char *rsc_provider = NULL;
 648     const char *rsc_type = NULL;
 649     bool cib_only = false;
 650     pcmk_resource_t *rsc = pe_find_resource(scheduler->resources, rsc_id);
 651 
 652     if (rsc == NULL) {
 653         out->err(out, "Resource %s not found", rsc_id);
 654         return ENXIO;
 655 
 656     } else if (!pcmk__is_primitive(rsc)) {
 657         out->err(out, "We can only process primitive resources, not %s", rsc_id);
 658         return EINVAL;
 659     }
 660 
 661     rsc_class = crm_element_value(rsc->xml, PCMK_XA_CLASS);
 662     rsc_provider = crm_element_value(rsc->xml, PCMK_XA_PROVIDER),
 663     rsc_type = crm_element_value(rsc->xml, PCMK_XA_TYPE);
 664     if ((rsc_class == NULL) || (rsc_type == NULL)) {
 665         out->err(out, "Resource %s does not have a class and type", rsc_id);
 666         return EINVAL;
 667     }
 668 
 669     {
 670         pcmk_node_t *node = pcmk_find_node(scheduler, host_uname);
 671 
 672         if (node == NULL) {
 673             out->err(out, "Node %s not found", host_uname);
 674             return pcmk_rc_node_unknown;
 675         }
 676 
 677         if (!(node->details->online)) {
 678             if (do_fail_resource) {
 679                 out->err(out, "Node %s is not online", host_uname);
 680                 return ENOTCONN;
 681             } else {
 682                 cib_only = true;
 683             }
 684         }
 685         if (!cib_only && pcmk__is_pacemaker_remote_node(node)) {
 686             node = pcmk__current_node(node->details->remote_rsc);
 687             if (node == NULL) {
 688                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
 689                          host_uname);
 690                 return ENOTCONN;
 691             }
 692             router_node = node->details->uname;
 693         }
 694     }
 695 
 696     if (rsc->clone_name) {
 697         rsc_api_id = rsc->clone_name;
 698         rsc_long_id = rsc->id;
 699     } else {
 700         rsc_api_id = rsc->id;
 701     }
 702     if (do_fail_resource) {
 703         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
 704                                       rsc_api_id, rsc_long_id,
 705                                       rsc_class, rsc_provider, rsc_type);
 706     } else {
 707         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
 708                                          rsc_api_id, rsc_long_id, rsc_class,
 709                                          rsc_provider, rsc_type, cib_only);
 710     }
 711 }
 712 
 713 /*!
 714  * \internal
 715  * \brief Get resource name as used in failure-related node attributes
 716  *
 717  * \param[in] rsc  Resource to check
 718  *
 719  * \return Newly allocated string containing resource's fail name
 720  * \note The caller is responsible for freeing the result.
 721  */
 722 static inline char *
 723 rsc_fail_name(const pcmk_resource_t *rsc)
     /*  */
 724 {
 725     const char *name = (rsc->clone_name? rsc->clone_name : rsc->id);
 726 
 727     if (pcmk_is_set(rsc->flags, pcmk_rsc_unique)) {
 728         return strdup(name);
 729     }
 730     return clone_strip(name);
 731 }
 732 
 733 // \return Standard Pacemaker return code
 734 static int
 735 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 736                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 737 {
 738     int rc = pcmk_rc_ok;
 739 
 740     /* Erase the resource's entire LRM history in the CIB, even if we're only
 741      * clearing a single operation's fail count. If we erased only entries for a
 742      * single operation, we might wind up with a wrong idea of the current
 743      * resource state, and we might not re-probe the resource.
 744      */
 745     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
 746     if (rc != pcmk_rc_ok) {
 747         return rc;
 748     }
 749 
 750     crm_trace("Processing %d mainloop inputs",
 751               pcmk_controld_api_replies_expected(controld_api));
 752     while (g_main_context_iteration(NULL, FALSE)) {
 753         crm_trace("Processed mainloop input, %d still remaining",
 754                   pcmk_controld_api_replies_expected(controld_api));
 755     }
 756     return rc;
 757 }
 758 
 759 // \return Standard Pacemaker return code
 760 static int
 761 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     /*  */
 762                    const char *node_name, const char *rsc_id, const char *operation,
 763                    const char *interval_spec, pcmk_scheduler_t *scheduler)
 764 {
 765     int rc = pcmk_rc_ok;
 766     const char *failed_value = NULL;
 767     const char *failed_id = NULL;
 768     char *interval_ms_s = NULL;
 769     GHashTable *rscs = NULL;
 770     GHashTableIter iter;
 771 
 772     /* Create a hash table to use as a set of resources to clean. This lets us
 773      * clean each resource only once (per node) regardless of how many failed
 774      * operations it has.
 775      */
 776     rscs = pcmk__strkey_table(NULL, NULL);
 777 
 778     // Normalize interval to milliseconds for comparison to history entry
 779     if (operation) {
 780         guint interval_ms = 0U;
 781 
 782         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 783         interval_ms_s = crm_strdup_printf("%u", interval_ms);
 784     }
 785 
 786     for (xmlNode *xml_op = pcmk__xe_first_child(scheduler->failed, NULL, NULL,
 787                                                 NULL);
 788          xml_op != NULL; xml_op = pcmk__xe_next(xml_op)) {
 789 
 790         failed_id = crm_element_value(xml_op, PCMK__XA_RSC_ID);
 791         if (failed_id == NULL) {
 792             // Malformed history entry, should never happen
 793             continue;
 794         }
 795 
 796         // No resource specified means all resources match
 797         if (rsc_id) {
 798             pcmk_resource_t *fail_rsc = NULL;
 799 
 800             fail_rsc = pe_find_resource_with_flags(scheduler->resources,
 801                                                    failed_id,
 802                                                    pcmk_rsc_match_history
 803                                                    |pcmk_rsc_match_anon_basename);
 804             if (!fail_rsc || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_casei)) {
 805                 continue;
 806             }
 807         }
 808 
 809         // Host name should always have been provided by this point
 810         failed_value = crm_element_value(xml_op, PCMK_XA_UNAME);
 811         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
 812             continue;
 813         }
 814 
 815         // No operation specified means all operations match
 816         if (operation) {
 817             failed_value = crm_element_value(xml_op, PCMK_XA_OPERATION);
 818             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
 819                 continue;
 820             }
 821 
 822             // Interval (if operation was specified) defaults to 0 (not all)
 823             failed_value = crm_element_value(xml_op, PCMK_META_INTERVAL);
 824             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
 825                 continue;
 826             }
 827         }
 828 
 829         g_hash_table_add(rscs, (gpointer) failed_id);
 830     }
 831 
 832     free(interval_ms_s);
 833 
 834     g_hash_table_iter_init(&iter, rscs);
 835     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
 836         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
 837         rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
 838         if (rc != pcmk_rc_ok) {
 839             return rc;
 840         }
 841     }
 842     g_hash_table_destroy(rscs);
 843     return rc;
 844 }
 845 
 846 // \return Standard Pacemaker return code
 847 static int
 848 clear_rsc_fail_attrs(const pcmk_resource_t *rsc, const char *operation,
     /*  */
 849                      const char *interval_spec, const pcmk_node_t *node)
 850 {
 851     int rc = pcmk_rc_ok;
 852     int attr_options = pcmk__node_attr_none;
 853     char *rsc_name = rsc_fail_name(rsc);
 854 
 855     if (pcmk__is_pacemaker_remote_node(node)) {
 856         attr_options |= pcmk__node_attr_remote;
 857     }
 858 
 859     rc = pcmk__attrd_api_clear_failures(NULL, node->details->uname, rsc_name,
 860                                         operation, interval_spec, NULL,
 861                                         attr_options);
 862     free(rsc_name);
 863     return rc;
 864 }
 865 
 866 // \return Standard Pacemaker return code
 867 int
 868 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 869                     const pcmk_resource_t *rsc, const char *operation,
 870                     const char *interval_spec, bool just_failures,
 871                     pcmk_scheduler_t *scheduler, gboolean force)
 872 {
 873     pcmk__output_t *out = scheduler->priv;
 874     int rc = pcmk_rc_ok;
 875     pcmk_node_t *node = NULL;
 876 
 877     if (rsc == NULL) {
 878         return ENXIO;
 879 
 880     } else if (rsc->children) {
 881 
 882         for (const GList *lpc = rsc->children; lpc != NULL; lpc = lpc->next) {
 883             const pcmk_resource_t *child = (const pcmk_resource_t *) lpc->data;
 884 
 885             rc = cli_resource_delete(controld_api, host_uname, child, operation,
 886                                      interval_spec, just_failures, scheduler,
 887                                      force);
 888             if (rc != pcmk_rc_ok) {
 889                 return rc;
 890             }
 891         }
 892         return pcmk_rc_ok;
 893 
 894     } else if (host_uname == NULL) {
 895         GList *lpc = NULL;
 896         GList *nodes = g_hash_table_get_values(rsc->known_on);
 897 
 898         if(nodes == NULL && force) {
 899             nodes = pcmk__copy_node_list(scheduler->nodes, false);
 900 
 901         } else if(nodes == NULL && rsc->exclusive_discover) {
 902             GHashTableIter iter;
 903             pcmk_node_t *node = NULL;
 904 
 905             g_hash_table_iter_init(&iter, rsc->allowed_nodes);
 906             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
 907                 if(node->weight >= 0) {
 908                     nodes = g_list_prepend(nodes, node);
 909                 }
 910             }
 911 
 912         } else if(nodes == NULL) {
 913             nodes = g_hash_table_get_values(rsc->allowed_nodes);
 914         }
 915 
 916         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
 917             node = (pcmk_node_t *) lpc->data;
 918 
 919             if (node->details->online) {
 920                 rc = cli_resource_delete(controld_api, node->details->uname, rsc,
 921                                          operation, interval_spec, just_failures,
 922                                          scheduler, force);
 923             }
 924             if (rc != pcmk_rc_ok) {
 925                 g_list_free(nodes);
 926                 return rc;
 927             }
 928         }
 929 
 930         g_list_free(nodes);
 931         return pcmk_rc_ok;
 932     }
 933 
 934     node = pcmk_find_node(scheduler, host_uname);
 935 
 936     if (node == NULL) {
 937         out->err(out, "Unable to clean up %s because node %s not found",
 938                  rsc->id, host_uname);
 939         return ENODEV;
 940     }
 941 
 942     if (!node->details->rsc_discovery_enabled) {
 943         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
 944                  rsc->id, host_uname);
 945         return EOPNOTSUPP;
 946     }
 947 
 948     if (controld_api == NULL) {
 949         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
 950                  rsc->id, host_uname);
 951         return pcmk_rc_ok;
 952     }
 953 
 954     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
 955     if (rc != pcmk_rc_ok) {
 956         out->err(out, "Unable to clean up %s failures on %s: %s",
 957                  rsc->id, host_uname, pcmk_rc_str(rc));
 958         return rc;
 959     }
 960 
 961     if (just_failures) {
 962         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
 963                                 interval_spec, scheduler);
 964     } else {
 965         rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
 966     }
 967     if (rc != pcmk_rc_ok) {
 968         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
 969                  rsc->id, host_uname, pcmk_rc_str(rc));
 970     } else {
 971         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
 972     }
 973     return rc;
 974 }
 975 
 976 // \return Standard Pacemaker return code
 977 int
 978 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
     /*  */
 979                 const char *operation, const char *interval_spec,
 980                 pcmk_scheduler_t *scheduler)
 981 {
 982     pcmk__output_t *out = scheduler->priv;
 983     int rc = pcmk_rc_ok;
 984     int attr_options = pcmk__node_attr_none;
 985     const char *display_name = node_name? node_name : "all nodes";
 986 
 987     if (controld_api == NULL) {
 988         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
 989                   display_name);
 990         return rc;
 991     }
 992 
 993     if (node_name) {
 994         pcmk_node_t *node = pcmk_find_node(scheduler, node_name);
 995 
 996         if (node == NULL) {
 997             out->err(out, "Unknown node: %s", node_name);
 998             return ENXIO;
 999         }
1000         if (pcmk__is_pacemaker_remote_node(node)) {
1001             attr_options |= pcmk__node_attr_remote;
1002         }
1003     }
1004 
1005     rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
1006                                         interval_spec, NULL, attr_options);
1007     if (rc != pcmk_rc_ok) {
1008         out->err(out, "Unable to clean up all failures on %s: %s",
1009                  display_name, pcmk_rc_str(rc));
1010         return rc;
1011     }
1012 
1013     if (node_name) {
1014         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
1015                                 operation, interval_spec, scheduler);
1016         if (rc != pcmk_rc_ok) {
1017             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
1018                      node_name, pcmk_rc_str(rc));
1019             return rc;
1020         }
1021     } else {
1022         for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
1023             pcmk_node_t *node = (pcmk_node_t *) iter->data;
1024 
1025             rc = clear_rsc_failures(out, controld_api, node->details->uname, NULL,
1026                                     operation, interval_spec, scheduler);
1027             if (rc != pcmk_rc_ok) {
1028                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
1029                          pcmk_rc_str(rc));
1030                 return rc;
1031             }
1032         }
1033     }
1034 
1035     out->info(out, "Cleaned up all resources on %s", display_name);
1036     return rc;
1037 }
1038 
1039 static void
1040 check_role(resource_checks_t *checks)
     /*  */
1041 {
1042     const char *role_s = g_hash_table_lookup(checks->rsc->meta,
1043                                              PCMK_META_TARGET_ROLE);
1044 
1045     if (role_s == NULL) {
1046         return;
1047     }
1048     switch (pcmk_parse_role(role_s)) {
1049         case pcmk_role_stopped:
1050             checks->flags |= rsc_remain_stopped;
1051             break;
1052 
1053         case pcmk_role_unpromoted:
1054             if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
1055                             pcmk_rsc_promotable)) {
1056                 checks->flags |= rsc_unpromotable;
1057             }
1058             break;
1059 
1060         default:
1061             break;
1062     }
1063 }
1064 
1065 static void
1066 check_managed(resource_checks_t *checks)
     /*  */
1067 {
1068     const char *managed_s = g_hash_table_lookup(checks->rsc->meta,
1069                                                 PCMK_META_IS_MANAGED);
1070 
1071     if ((managed_s != NULL) && !crm_is_true(managed_s)) {
1072         checks->flags |= rsc_unmanaged;
1073     }
1074 }
1075 
1076 static void
1077 check_locked(resource_checks_t *checks)
     /*  */
1078 {
1079     if (checks->rsc->lock_node != NULL) {
1080         checks->flags |= rsc_locked;
1081         checks->lock_node = checks->rsc->lock_node->details->uname;
1082     }
1083 }
1084 
1085 static bool
1086 node_is_unhealthy(pcmk_node_t *node)
     /*  */
1087 {
1088     switch (pe__health_strategy(node->details->data_set)) {
1089         case pcmk__health_strategy_none:
1090             break;
1091 
1092         case pcmk__health_strategy_no_red:
1093             if (pe__node_health(node) < 0) {
1094                 return true;
1095             }
1096             break;
1097 
1098         case pcmk__health_strategy_only_green:
1099             if (pe__node_health(node) <= 0) {
1100                 return true;
1101             }
1102             break;
1103 
1104         case pcmk__health_strategy_progressive:
1105         case pcmk__health_strategy_custom:
1106             /* @TODO These are finite scores, possibly with rules, and possibly
1107              * combining with other scores, so attributing these as a cause is
1108              * nontrivial.
1109              */
1110             break;
1111     }
1112     return false;
1113 }
1114 
1115 static void
1116 check_node_health(resource_checks_t *checks, pcmk_node_t *node)
     /*  */
1117 {
1118     if (node == NULL) {
1119         GHashTableIter iter;
1120         bool allowed = false;
1121         bool all_nodes_unhealthy = true;
1122 
1123         g_hash_table_iter_init(&iter, checks->rsc->allowed_nodes);
1124         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
1125             allowed = true;
1126             if (!node_is_unhealthy(node)) {
1127                 all_nodes_unhealthy = false;
1128                 break;
1129             }
1130         }
1131         if (allowed && all_nodes_unhealthy) {
1132             checks->flags |= rsc_node_health;
1133         }
1134 
1135     } else if (node_is_unhealthy(node)) {
1136         checks->flags |= rsc_node_health;
1137     }
1138 }
1139 
1140 int
1141 cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
     /*  */
1142 {
1143     resource_checks_t checks = { .rsc = rsc };
1144 
1145     check_role(&checks);
1146     check_managed(&checks);
1147     check_locked(&checks);
1148     check_node_health(&checks, node);
1149 
1150     return out->message(out, "resource-check-list", &checks);
1151 }
1152 
1153 // \return Standard Pacemaker return code
1154 int
1155 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
1156                   const char *rsc_id, pcmk_scheduler_t *scheduler)
1157 {
1158     crm_notice("Failing %s on %s", rsc_id, host_uname);
1159     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
1160 }
1161 
1162 static GHashTable *
1163 generate_resource_params(pcmk_resource_t *rsc, pcmk_node_t *node,
     /*  */
1164                          pcmk_scheduler_t *scheduler)
1165 {
1166     GHashTable *params = NULL;
1167     GHashTable *meta = NULL;
1168     GHashTable *combined = NULL;
1169     GHashTableIter iter;
1170     char *key = NULL;
1171     char *value = NULL;
1172 
1173     combined = pcmk__strkey_table(free, free);
1174 
1175     params = pe_rsc_params(rsc, node, scheduler);
1176     if (params != NULL) {
1177         g_hash_table_iter_init(&iter, params);
1178         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1179             pcmk__insert_dup(combined, key, value);
1180         }
1181     }
1182 
1183     meta = pcmk__strkey_table(free, free);
1184     get_meta_attributes(meta, rsc, NULL, scheduler);
1185     if (meta != NULL) {
1186         g_hash_table_iter_init(&iter, meta);
1187         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1188             char *crm_name = crm_meta_name(key);
1189 
1190             g_hash_table_insert(combined, crm_name, strdup(value));
1191         }
1192         g_hash_table_destroy(meta);
1193     }
1194 
1195     return combined;
1196 }
1197 
1198 bool resource_is_running_on(pcmk_resource_t *rsc, const char *host)
     /*  */
1199 {
1200     bool found = true;
1201     GList *hIter = NULL;
1202     GList *hosts = NULL;
1203 
1204     if (rsc == NULL) {
1205         return false;
1206     }
1207 
1208     rsc->fns->location(rsc, &hosts, TRUE);
1209     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
1210         pcmk_node_t *node = (pcmk_node_t *) hIter->data;
1211 
1212         if (pcmk__strcase_any_of(host, node->details->uname, node->details->id, NULL)) {
1213             crm_trace("Resource %s is running on %s\n", rsc->id, host);
1214             goto done;
1215         }
1216     }
1217 
1218     if (host != NULL) {
1219         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
1220         found = false;
1221 
1222     } else if(host == NULL && hosts == NULL) {
1223         crm_trace("Resource %s is not running\n", rsc->id);
1224         found = false;
1225     }
1226 
1227   done:
1228     g_list_free(hosts);
1229     return found;
1230 }
1231 
1232 /*!
1233  * \internal
1234  * \brief Create a list of all resources active on host from a given list
1235  *
1236  * \param[in] host      Name of host to check whether resources are active
1237  * \param[in] rsc_list  List of resources to check
1238  *
1239  * \return New list of resources from list that are active on host
1240  */
1241 static GList *
1242 get_active_resources(const char *host, GList *rsc_list)
     /*  */
1243 {
1244     GList *rIter = NULL;
1245     GList *active = NULL;
1246 
1247     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
1248         pcmk_resource_t *rsc = (pcmk_resource_t *) rIter->data;
1249 
1250         /* Expand groups to their members, because if we're restarting a member
1251          * other than the first, we can't otherwise tell which resources are
1252          * stopping and starting.
1253          */
1254         if (pcmk__is_group(rsc)) {
1255             active = g_list_concat(active,
1256                                    get_active_resources(host, rsc->children));
1257         } else if (resource_is_running_on(rsc, host)) {
1258             active = g_list_append(active, strdup(rsc->id));
1259         }
1260     }
1261     return active;
1262 }
1263 
1264 static void dump_list(GList *items, const char *tag)
     /*  */
1265 {
1266     int lpc = 0;
1267     GList *item = NULL;
1268 
1269     for (item = items; item != NULL; item = item->next) {
1270         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
1271         lpc++;
1272     }
1273 }
1274 
1275 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
     /*  */
1276 {
1277     GList *item = NULL;
1278 
1279     for (item = items; item != NULL; item = item->next) {
1280         out->info(out, "%s%s", tag, (const char *)item->data);
1281     }
1282 }
1283 
1284 /*!
1285  * \internal
1286  * \brief Upgrade XML to latest schema version and use it as scheduler input
1287  *
1288  * This also updates the scheduler timestamp to the current time.
1289  *
1290  * \param[in,out] scheduler  Scheduler data to update
1291  * \param[in,out] xml        XML to use as input
1292  *
1293  * \return Standard Pacemaker return code
1294  * \note On success, caller is responsible for freeing memory allocated for
1295  *       scheduler->now.
1296  */
1297 int
1298 update_scheduler_input(pcmk_scheduler_t *scheduler, xmlNode **xml)
     /*  */
1299 {
1300     int rc = pcmk__update_configured_schema(xml, false);
1301 
1302     if (rc == pcmk_rc_ok) {
1303         scheduler->input = *xml;
1304         scheduler->now = crm_time_new(NULL);
1305     }
1306     return rc;
1307 }
1308 
1309 /*!
1310  * \internal
1311  * \brief Update scheduler XML input based on a CIB query
1312  *
1313  * \param[in] scheduler  Scheduler data to initialize
1314  * \param[in] cib        Connection to the CIB manager
1315  *
1316  * \return Standard Pacemaker return code
1317  * \note On success, caller is responsible for freeing memory allocated for
1318  *       scheduler->input and scheduler->now.
1319  */
1320 static int
1321 update_scheduler_input_to_cib(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
     /*  */
1322                               cib_t *cib)
1323 {
1324     xmlNode *cib_xml_copy = NULL;
1325     int rc = pcmk_rc_ok;
1326 
1327     rc = cib->cmds->query(cib, NULL, &cib_xml_copy, cib_scope_local | cib_sync_call);
1328     rc = pcmk_legacy2rc(rc);
1329 
1330     if (rc != pcmk_rc_ok) {
1331         out->err(out, "Could not obtain the current CIB: %s (%d)", pcmk_rc_str(rc), rc);
1332         return rc;
1333     }
1334     rc = update_scheduler_input(scheduler, &cib_xml_copy);
1335     if (rc != pcmk_rc_ok) {
1336         out->err(out, "Could not upgrade the current CIB XML");
1337         free_xml(cib_xml_copy);
1338         return rc;
1339     }
1340 
1341     return rc;
1342 }
1343 
1344 // \return Standard Pacemaker return code
1345 static int
1346 update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, bool simulate)
     /*  */
1347 {
1348     char *pid = NULL;
1349     char *shadow_file = NULL;
1350     cib_t *shadow_cib = NULL;
1351     int rc = pcmk_rc_ok;
1352 
1353     pcmk__output_t *out = scheduler->priv;
1354 
1355     pe_reset_working_set(scheduler);
1356     pcmk__set_scheduler_flags(scheduler,
1357                               pcmk_sched_no_counts|pcmk_sched_no_compat);
1358     rc = update_scheduler_input_to_cib(out, scheduler, cib);
1359     if (rc != pcmk_rc_ok) {
1360         return rc;
1361     }
1362 
1363     if(simulate) {
1364         bool prev_quiet = false;
1365 
1366         pid = pcmk__getpid_s();
1367         shadow_cib = cib_shadow_new(pid);
1368         shadow_file = get_shadow_file(pid);
1369 
1370         if (shadow_cib == NULL) {
1371             out->err(out, "Could not create shadow cib: '%s'", pid);
1372             rc = ENXIO;
1373             goto done;
1374         }
1375 
1376         rc = pcmk__xml_write_file(scheduler->input, shadow_file, false, NULL);
1377         if (rc != pcmk_rc_ok) {
1378             out->err(out, "Could not populate shadow cib: %s", pcmk_rc_str(rc));
1379             goto done;
1380         }
1381 
1382         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
1383         rc = pcmk_legacy2rc(rc);
1384 
1385         if (rc != pcmk_rc_ok) {
1386             out->err(out, "Could not connect to shadow cib: %s",
1387                      pcmk_rc_str(rc));
1388             goto done;
1389         }
1390 
1391         pcmk__schedule_actions(scheduler->input,
1392                                pcmk_sched_no_counts|pcmk_sched_no_compat,
1393                                scheduler);
1394 
1395         prev_quiet = out->is_quiet(out);
1396         out->quiet = true;
1397         pcmk__simulate_transition(scheduler, shadow_cib, NULL);
1398         out->quiet = prev_quiet;
1399 
1400         rc = update_dataset(shadow_cib, scheduler, false);
1401 
1402     } else {
1403         cluster_status(scheduler);
1404     }
1405 
1406   done:
1407     // Do not free scheduler->input here, we need rsc->xml to be valid later on
1408     cib_delete(shadow_cib);
1409     free(pid);
1410 
1411     if(shadow_file) {
1412         unlink(shadow_file);
1413         free(shadow_file);
1414     }
1415 
1416     return rc;
1417 }
1418 
1419 /*!
1420  * \internal
1421  * \brief Find the maximum stop timeout of a resource and its children (if any)
1422  *
1423  * \param[in,out] rsc  Resource to get timeout for
1424  *
1425  * \return Maximum stop timeout for \p rsc (in milliseconds)
1426  */
1427 static guint
1428 max_rsc_stop_timeout(pcmk_resource_t *rsc)
     /*  */
1429 {
1430     long long result_ll;
1431     guint max_delay = 0;
1432     xmlNode *config = NULL;
1433     GHashTable *meta = NULL;
1434 
1435     if (rsc == NULL) {
1436         return 0;
1437     }
1438 
1439     // If resource is collective, use maximum of its children's stop timeouts
1440     if (rsc->children != NULL) {
1441         for (GList *iter = rsc->children; iter; iter = iter->next) {
1442             pcmk_resource_t *child = iter->data;
1443             guint delay = max_rsc_stop_timeout(child);
1444 
1445             if (delay > max_delay) {
1446                 pcmk__rsc_trace(rsc,
1447                                 "Maximum stop timeout for %s is now %s "
1448                                 "due to %s", rsc->id,
1449                                 pcmk__readable_interval(delay), child->id);
1450                 max_delay = delay;
1451             }
1452         }
1453         return max_delay;
1454     }
1455 
1456     // Get resource's stop action configuration from CIB
1457     config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);
1458 
1459     /* Get configured timeout for stop action (fully evaluated for rules,
1460      * defaults, etc.).
1461      *
1462      * @TODO This currently ignores node (which might matter for rules)
1463      */
1464     meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
1465     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
1466                        &result_ll, -1LL) == pcmk_rc_ok) && (result_ll >= 0)) {
1467         max_delay = (guint) QB_MIN(result_ll, UINT_MAX);
1468     }
1469     g_hash_table_destroy(meta);
1470 
1471     return max_delay;
1472 }
1473 
1474 /*!
1475  * \internal
1476  * \brief Find a reasonable waiting time for stopping any one resource in a list
1477  *
1478  * \param[in,out] scheduler  Scheduler data
1479  * \param[in]     resources  List of names of resources that will be stopped
1480  *
1481  * \return Rough estimate of a reasonable time to wait (in seconds) to stop any
1482  *         one resource in \p resources
1483  * \note This estimate is very rough, simply the maximum stop timeout of all
1484  *       given resources and their children, plus a small fudge factor. It does
1485  *       not account for children that must be stopped in sequence, action
1486  *       throttling, or any demotions needed. It checks the stop timeout, even
1487  *       if the resources in question are actually being started.
1488  */
1489 static guint
1490 wait_time_estimate(pcmk_scheduler_t *scheduler, const GList *resources)
     /*  */
1491 {
1492     guint max_delay = 0U;
1493 
1494     // Find maximum stop timeout in milliseconds
1495     for (const GList *item = resources; item != NULL; item = item->next) {
1496         pcmk_resource_t *rsc = pe_find_resource(scheduler->resources,
1497                                                 (const char *) item->data);
1498         guint delay = max_rsc_stop_timeout(rsc);
1499 
1500         if (delay > max_delay) {
1501             pcmk__rsc_trace(rsc,
1502                             "Wait time is now %s due to %s",
1503                             pcmk__readable_interval(delay), rsc->id);
1504             max_delay = delay;
1505         }
1506     }
1507 
1508     return (max_delay / 1000U) + 5U;
1509 }
1510 
1511 #define waiting_for_starts(d, r, h) ((d != NULL) || \
1512                                     (!resource_is_running_on((r), (h))))
1513 
1514 /*!
1515  * \internal
1516  * \brief Restart a resource (on a particular host if requested).
1517  *
1518  * \param[in,out] out                 Output object
1519  * \param[in,out] rsc                 The resource to restart
1520  * \param[in]     node                Node to restart resource on (NULL for all)
1521  * \param[in]     move_lifetime       If not NULL, how long constraint should
1522  *                                    remain in effect (as ISO 8601 string)
1523  * \param[in]     timeout_ms          Consider failed if actions do not complete
1524  *                                    in this time (specified in milliseconds,
1525  *                                    but a two-second granularity is actually
1526  *                                    used; if 0, it will be calculated based on
1527  *                                    the resource timeout)
1528  * \param[in,out] cib                 Connection to the CIB manager
1529  * \param[in]     cib_options         Group of enum cib_call_options flags to
1530  *                                    use with CIB calls
1531  * \param[in]     promoted_role_only  If true, limit to promoted instances
1532  * \param[in]     force               If true, apply only to requested instance
1533  *                                    if part of a collective resource
1534  *
1535  * \return Standard Pacemaker return code (exits on certain failures)
1536  */
1537 int
1538 cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
1539                      const pcmk_node_t *node, const char *move_lifetime,
1540                      guint timeout_ms, cib_t *cib, int cib_options,
1541                      gboolean promoted_role_only, gboolean force)
1542 {
1543     int rc = pcmk_rc_ok;
1544     int lpc = 0;
1545     int before = 0;
1546     guint step_timeout_s = 0;
1547     guint sleep_interval = 2U;
1548     guint timeout = timeout_ms / 1000U;
1549 
1550     bool stop_via_ban = false;
1551     char *rsc_id = NULL;
1552     char *lookup_id = NULL;
1553     char *orig_target_role = NULL;
1554 
1555     GList *list_delta = NULL;
1556     GList *target_active = NULL;
1557     GList *current_active = NULL;
1558     GList *restart_target_active = NULL;
1559 
1560     pcmk_scheduler_t *scheduler = NULL;
1561     pcmk_resource_t *parent = uber_parent(rsc);
1562 
1563     bool running = false;
1564     const char *id = rsc->clone_name ? rsc->clone_name : rsc->id;
1565     const char *host = node ? node->details->uname : NULL;
1566 
1567     /* If the implicit resource or primitive resource of a bundle is given, operate on the
1568      * bundle itself instead.
1569      */
1570     if (pcmk__is_bundled(rsc)) {
1571         rsc = parent->parent;
1572     }
1573 
1574     running = resource_is_running_on(rsc, host);
1575 
1576     if (pcmk__is_clone(parent) && !running) {
1577         if (pcmk__is_unique_clone(parent)) {
1578             lookup_id = strdup(rsc->id);
1579         } else {
1580             lookup_id = clone_strip(rsc->id);
1581         }
1582 
1583         rsc = parent->fns->find_rsc(parent, lookup_id, node,
1584                                     pcmk_rsc_match_basename
1585                                     |pcmk_rsc_match_current_node);
1586         free(lookup_id);
1587         running = resource_is_running_on(rsc, host);
1588     }
1589 
1590     if (!running) {
1591         if (host) {
1592             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
1593         } else {
1594             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
1595         }
1596         return ENXIO;
1597     }
1598 
1599     if (!pcmk_is_set(rsc->flags, pcmk_rsc_managed)) {
1600         out->err(out, "Unmanaged resources cannot be restarted.");
1601         return EAGAIN;
1602     }
1603 
1604     rsc_id = strdup(rsc->id);
1605 
1606     if (pcmk__is_unique_clone(parent)) {
1607         lookup_id = strdup(rsc->id);
1608     } else {
1609         lookup_id = clone_strip(rsc->id);
1610     }
1611 
1612     if (host) {
1613         if (pcmk__is_clone(rsc) || pe_bundle_replicas(rsc)) {
1614             stop_via_ban = true;
1615         } else if (pcmk__is_clone(parent)) {
1616             stop_via_ban = true;
1617             free(lookup_id);
1618             lookup_id = strdup(parent->id);
1619         }
1620     }
1621 
1622     /*
1623       grab full cib
1624       determine originally active resources
1625       disable or ban
1626       poll cib and watch for affected resources to get stopped
1627       without --timeout, calculate the stop timeout for each step and wait for that
1628       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
1629       if everything stopped, re-enable or un-ban
1630       poll cib and watch for affected resources to get started
1631       without --timeout, calculate the start timeout for each step and wait for that
1632       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
1633       report success
1634 
1635       Optimizations:
1636       - use constraints to determine ordered list of affected resources
1637       - Allow a --no-deps option (aka. --force-restart)
1638     */
1639 
1640     scheduler = pe_new_working_set();
1641     if (scheduler == NULL) {
1642         rc = errno;
1643         out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
1644         goto done;
1645     }
1646 
1647     scheduler->priv = out;
1648     rc = update_dataset(cib, scheduler, false);
1649 
1650     if(rc != pcmk_rc_ok) {
1651         out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
1652         goto done;
1653     }
1654 
1655     restart_target_active = get_active_resources(host, scheduler->resources);
1656     current_active = get_active_resources(host, scheduler->resources);
1657 
1658     dump_list(current_active, "Origin");
1659 
1660     if (stop_via_ban) {
1661         /* Stop the clone or bundle instance by banning it from the host */
1662         out->quiet = true;
1663         rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
1664                               cib_options, promoted_role_only,
1665                               PCMK_ROLE_PROMOTED);
1666     } else {
1667         xmlNode *xml_search = NULL;
1668 
1669         /* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
1670          * Remember any existing PCMK_META_TARGET_ROLE so we can restore it
1671          * later (though it only makes any difference if it's Unpromoted).
1672          */
1673 
1674         rc = find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
1675                                 PCMK_META_TARGET_ROLE, &xml_search);
1676 
1677         if (rc == pcmk_rc_ok) {
1678             orig_target_role = crm_element_value_copy(xml_search, PCMK_XA_VALUE);
1679         }
1680 
1681         free_xml(xml_search);
1682 
1683         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
1684                                            PCMK_XE_META_ATTRIBUTES, NULL,
1685                                            PCMK_META_TARGET_ROLE,
1686                                            PCMK_ACTION_STOPPED, FALSE, cib,
1687                                            force);
1688     }
1689     if(rc != pcmk_rc_ok) {
1690         out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
1691                  rsc_id, pcmk_rc_str(rc), rc);
1692         if (current_active != NULL) {
1693             g_list_free_full(current_active, free);
1694             current_active = NULL;
1695         }
1696         if (restart_target_active != NULL) {
1697             g_list_free_full(restart_target_active, free);
1698             restart_target_active = NULL;
1699         }
1700         goto done;
1701     }
1702 
1703     rc = update_dataset(cib, scheduler, true);
1704     if(rc != pcmk_rc_ok) {
1705         out->err(out, "Could not determine which resources would be stopped");
1706         goto failure;
1707     }
1708 
1709     target_active = get_active_resources(host, scheduler->resources);
1710     dump_list(target_active, "Target");
1711 
1712     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1713     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
1714     display_list(out, list_delta, " * ");
1715 
1716     step_timeout_s = timeout / sleep_interval;
1717     while (list_delta != NULL) {
1718         before = g_list_length(list_delta);
1719         if(timeout_ms == 0) {
1720             step_timeout_s = wait_time_estimate(scheduler, list_delta)
1721                              / sleep_interval;
1722         }
1723 
1724         /* We probably don't need the entire step timeout */
1725         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
1726             sleep(sleep_interval);
1727             if(timeout) {
1728                 timeout -= sleep_interval;
1729                 crm_trace("%us remaining", timeout);
1730             }
1731             rc = update_dataset(cib, scheduler, FALSE);
1732             if(rc != pcmk_rc_ok) {
1733                 out->err(out, "Could not determine which resources were stopped");
1734                 goto failure;
1735             }
1736 
1737             if (current_active != NULL) {
1738                 g_list_free_full(current_active, free);
1739             }
1740             current_active = get_active_resources(host, scheduler->resources);
1741 
1742             g_list_free(list_delta);
1743             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1744 
1745             dump_list(current_active, "Current");
1746             dump_list(list_delta, "Delta");
1747         }
1748 
1749         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
1750         if(before == g_list_length(list_delta)) {
1751             /* aborted during stop phase, print the contents of list_delta */
1752             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1753             display_list(out, list_delta, " * ");
1754             rc = ETIME;
1755             goto failure;
1756         }
1757 
1758     }
1759 
1760     if (stop_via_ban) {
1761         rc = cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
1762 
1763     } else if (orig_target_role) {
1764         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
1765                                            PCMK_XE_META_ATTRIBUTES, NULL,
1766                                            PCMK_META_TARGET_ROLE,
1767                                            orig_target_role, FALSE, cib, force);
1768         free(orig_target_role);
1769         orig_target_role = NULL;
1770     } else {
1771         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL,
1772                                            PCMK_XE_META_ATTRIBUTES, NULL,
1773                                            PCMK_META_TARGET_ROLE, cib,
1774                                            cib_options, force);
1775     }
1776 
1777     if(rc != pcmk_rc_ok) {
1778         out->err(out,
1779                  "Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
1780                  rsc_id, pcmk_rc_str(rc), rc);
1781         goto done;
1782     }
1783 
1784     if (target_active != NULL) {
1785         g_list_free_full(target_active, free);
1786     }
1787     target_active = restart_target_active;
1788 
1789     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1790     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
1791     display_list(out, list_delta, " * ");
1792 
1793     step_timeout_s = timeout / sleep_interval;
1794     while (waiting_for_starts(list_delta, rsc, host)) {
1795         before = g_list_length(list_delta);
1796         if(timeout_ms == 0) {
1797             step_timeout_s = wait_time_estimate(scheduler, list_delta)
1798                              / sleep_interval;
1799         }
1800 
1801         /* We probably don't need the entire step timeout */
1802         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
1803 
1804             sleep(sleep_interval);
1805             if(timeout) {
1806                 timeout -= sleep_interval;
1807                 crm_trace("%ds remaining", timeout);
1808             }
1809 
1810             rc = update_dataset(cib, scheduler, false);
1811             if(rc != pcmk_rc_ok) {
1812                 out->err(out, "Could not determine which resources were started");
1813                 goto failure;
1814             }
1815 
1816             /* It's OK if dependent resources moved to a different node,
1817              * so we check active resources on all nodes.
1818              */
1819             if (current_active != NULL) {
1820                 g_list_free_full(current_active, free);
1821             }
1822             current_active = get_active_resources(NULL, scheduler->resources);
1823 
1824             g_list_free(list_delta);
1825             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1826             dump_list(current_active, "Current");
1827             dump_list(list_delta, "Delta");
1828         }
1829 
1830         if(before == g_list_length(list_delta)) {
1831             /* aborted during start phase, print the contents of list_delta */
1832             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1833             display_list(out, list_delta, " * ");
1834             rc = ETIME;
1835             goto failure;
1836         }
1837 
1838     }
1839 
1840     rc = pcmk_rc_ok;
1841     goto done;
1842 
1843   failure:
1844     if (stop_via_ban) {
1845         cli_resource_clear(lookup_id, host, NULL, cib, cib_options, true, force);
1846     } else if (orig_target_role) {
1847         cli_resource_update_attribute(rsc, rsc_id, NULL,
1848                                       PCMK_XE_META_ATTRIBUTES, NULL,
1849                                       PCMK_META_TARGET_ROLE, orig_target_role,
1850                                       FALSE, cib, force);
1851         free(orig_target_role);
1852     } else {
1853         cli_resource_delete_attribute(rsc, rsc_id, NULL,
1854                                       PCMK_XE_META_ATTRIBUTES, NULL,
1855                                       PCMK_META_TARGET_ROLE, cib, cib_options,
1856                                       force);
1857     }
1858 
1859 done:
1860     if (list_delta != NULL) {
1861         g_list_free(list_delta);
1862     }
1863     if (current_active != NULL) {
1864         g_list_free_full(current_active, free);
1865     }
1866     if (target_active != NULL && (target_active != restart_target_active)) {
1867         g_list_free_full(target_active, free);
1868     }
1869     if (restart_target_active != NULL) {
1870         g_list_free_full(restart_target_active, free);
1871     }
1872     free(rsc_id);
1873     free(lookup_id);
1874     pe_free_working_set(scheduler);
1875     return rc;
1876 }
1877 
1878 static inline bool
1879 action_is_pending(const pcmk_action_t *action)
     /*  */
1880 {
1881     if (pcmk_any_flags_set(action->flags,
1882                            pcmk_action_optional|pcmk_action_pseudo)
1883         || !pcmk_is_set(action->flags, pcmk_action_runnable)
1884         || pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
1885         return false;
1886     }
1887     return true;
1888 }
1889 
1890 /*!
1891  * \internal
1892  * \brief Check whether any actions in a list are pending
1893  *
1894  * \param[in] actions   List of actions to check
1895  *
1896  * \return true if any actions in the list are pending, otherwise false
1897  */
1898 static bool
1899 actions_are_pending(const GList *actions)
     /*  */
1900 {
1901     for (const GList *action = actions; action != NULL; action = action->next) {
1902         const pcmk_action_t *a = (const pcmk_action_t *) action->data;
1903 
1904         if (action_is_pending(a)) {
1905             crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
1906             return true;
1907         }
1908     }
1909     return false;
1910 }
1911 
1912 static void
1913 print_pending_actions(pcmk__output_t *out, GList *actions)
     /*  */
1914 {
1915     GList *action;
1916 
1917     out->info(out, "Pending actions:");
1918     for (action = actions; action != NULL; action = action->next) {
1919         pcmk_action_t *a = (pcmk_action_t *) action->data;
1920 
1921         if (!action_is_pending(a)) {
1922             continue;
1923         }
1924 
1925         if (a->node) {
1926             out->info(out, "\tAction %d: %s\ton %s",
1927                       a->id, a->uuid, pcmk__node_name(a->node));
1928         } else {
1929             out->info(out, "\tAction %d: %s", a->id, a->uuid);
1930         }
1931     }
1932 }
1933 
1934 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
1935 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
1936 
1937 /* For --wait, how long to sleep between cluster state checks */
1938 #define WAIT_SLEEP_S (2)
1939 
1940 /*!
1941  * \internal
1942  * \brief Wait until all pending cluster actions are complete
1943  *
1944  * This waits until either the CIB's transition graph is idle or a timeout is
1945  * reached.
1946  *
1947  * \param[in,out] out          Output object
1948  * \param[in]     timeout_ms   Consider failed if actions do not complete in
1949  *                             this time (specified in milliseconds, but
1950  *                             one-second granularity is actually used; if 0, a
1951  *                             default will be used)
1952  * \param[in,out] cib          Connection to the CIB manager
1953  *
1954  * \return Standard Pacemaker return code
1955  */
1956 int
1957 wait_till_stable(pcmk__output_t *out, guint timeout_ms, cib_t * cib)
     /*  */
1958 {
1959     pcmk_scheduler_t *scheduler = NULL;
1960     xmlXPathObjectPtr search;
1961     int rc = pcmk_rc_ok;
1962     bool pending_unknown_state_resources;
1963     time_t expire_time = time(NULL);
1964     time_t time_diff;
1965     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
1966     char *xpath = NULL;
1967 
1968     if (timeout_ms == 0) {
1969         expire_time += WAIT_DEFAULT_TIMEOUT_S;
1970     } else {
1971         expire_time += (timeout_ms + 999) / 1000;
1972     }
1973 
1974     scheduler = pe_new_working_set();
1975     if (scheduler == NULL) {
1976         return ENOMEM;
1977     }
1978 
1979     xpath = crm_strdup_printf("/" PCMK_XE_CIB "/" PCMK_XE_STATUS
1980                               "/" PCMK__XE_NODE_STATE "/" PCMK__XE_LRM
1981                               "/" PCMK__XE_LRM_RESOURCES
1982                               "/" PCMK__XE_LRM_RESOURCE
1983                               "/" PCMK__XE_LRM_RSC_OP
1984                               "[@" PCMK__XA_RC_CODE "='%d']",
1985                               PCMK_OCF_UNKNOWN);
1986     do {
1987         /* Abort if timeout is reached */
1988         time_diff = expire_time - time(NULL);
1989         if (time_diff <= 0) {
1990             print_pending_actions(out, scheduler->actions);
1991             rc = ETIME;
1992             break;
1993         }
1994 
1995         crm_info("Waiting up to %lld seconds for cluster actions to complete",
1996                  (long long) time_diff);
1997 
1998         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
1999             sleep(WAIT_SLEEP_S);
2000         }
2001 
2002         /* Get latest transition graph */
2003         pe_reset_working_set(scheduler);
2004         rc = update_scheduler_input_to_cib(out, scheduler, cib);
2005         if (rc != pcmk_rc_ok) {
2006             break;
2007         }
2008         pcmk__schedule_actions(scheduler->input,
2009                                pcmk_sched_no_counts|pcmk_sched_no_compat,
2010                                scheduler);
2011 
2012         if (!printed_version_warning) {
2013             /* If the DC has a different version than the local node, the two
2014              * could come to different conclusions about what actions need to be
2015              * done. Warn the user in this case.
2016              *
2017              * @TODO A possible long-term solution would be to reimplement the
2018              * wait as a new controller operation that would be forwarded to the
2019              * DC. However, that would have potential problems of its own.
2020              */
2021             const char *dc_version = g_hash_table_lookup(scheduler->config_hash,
2022                                                          PCMK_OPT_DC_VERSION);
2023 
2024             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
2025                 out->info(out, "warning: wait option may not work properly in "
2026                           "mixed-version cluster");
2027                 printed_version_warning = true;
2028             }
2029         }
2030 
2031         search = xpath_search(scheduler->input, xpath);
2032         pending_unknown_state_resources = (numXpathResults(search) > 0);
2033         freeXpathObject(search);
2034     } while (actions_are_pending(scheduler->actions) || pending_unknown_state_resources);
2035 
2036     pe_free_working_set(scheduler);
2037     free(xpath);
2038     return rc;
2039 }
2040 
2041 static const char *
2042 get_action(const char *rsc_action) {
     /*  */
2043     const char *action = NULL;
2044 
2045     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
2046         action = PCMK_ACTION_VALIDATE_ALL;
2047 
2048     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
2049         action = PCMK_ACTION_MONITOR;
2050 
2051     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
2052                                     "force-demote", "force-promote", NULL)) {
2053         action = rsc_action+6;
2054     } else {
2055         action = rsc_action;
2056     }
2057 
2058     return action;
2059 }
2060 
2061 /*!
2062  * \brief Set up environment variables as expected by resource agents
2063  *
2064  * When the cluster executes resource agents, it adds certain environment
2065  * variables (directly or via resource meta-attributes) expected by some
2066  * resource agents. Add the essential ones that many resource agents expect, so
2067  * the behavior is the same for command-line execution.
2068  *
2069  * \param[in,out] params       Resource parameters that will be passed to agent
2070  * \param[in]     timeout_ms   Action timeout (in milliseconds)
2071  * \param[in]     check_level  OCF check level
2072  * \param[in]     verbosity    Verbosity level
2073  */
2074 static void
2075 set_agent_environment(GHashTable *params, guint timeout_ms, int check_level,
     /*  */
2076                       int verbosity)
2077 {
2078     g_hash_table_insert(params, crm_meta_name(PCMK_META_TIMEOUT),
2079                         crm_strdup_printf("%u", timeout_ms));
2080 
2081     pcmk__insert_dup(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
2082 
2083     if (check_level >= 0) {
2084         char *level = crm_strdup_printf("%d", check_level);
2085 
2086         setenv("OCF_CHECK_LEVEL", level, 1);
2087         free(level);
2088     }
2089 
2090     pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
2091     if (verbosity > 1) {
2092         setenv("OCF_TRACE_RA", "1", 1);
2093     }
2094 
2095     /* A resource agent using the standard ocf-shellfuncs library will not print
2096      * messages to stderr if it doesn't have a controlling terminal (e.g. if
2097      * crm_resource is called via script or ssh). This forces it to do so.
2098      */
2099     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
2100 }
2101 
2102 /*!
2103  * \internal
2104  * \brief Apply command-line overrides to resource parameters
2105  *
2106  * \param[in,out] params     Parameters to be passed to agent
2107  * \param[in]     overrides  Parameters to override (or NULL if none)
2108  */
2109 static void
2110 apply_overrides(GHashTable *params, GHashTable *overrides)
     /*  */
2111 {
2112     if (overrides != NULL) {
2113         GHashTableIter iter;
2114         char *name = NULL;
2115         char *value = NULL;
2116 
2117         g_hash_table_iter_init(&iter, overrides);
2118         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
2119                                       (gpointer *) &value)) {
2120             pcmk__insert_dup(params, name, value);
2121         }
2122     }
2123 }
2124 
2125 crm_exit_t
2126 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     /*  */
2127                                  const char *rsc_class, const char *rsc_prov,
2128                                  const char *rsc_type, const char *rsc_action,
2129                                  GHashTable *params, GHashTable *override_hash,
2130                                  guint timeout_ms, int resource_verbose,
2131                                  gboolean force, int check_level)
2132 {
2133     const char *class = rsc_class;
2134     const char *action = get_action(rsc_action);
2135     crm_exit_t exit_code = CRM_EX_OK;
2136     svc_action_t *op = NULL;
2137 
2138     // If no timeout was provided, use the same default as the cluster
2139     if (timeout_ms == 0U) {
2140         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
2141     }
2142 
2143     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
2144     apply_overrides(params, override_hash);
2145 
2146     op = services__create_resource_action(rsc_name? rsc_name : "test",
2147                                           rsc_class, rsc_prov, rsc_type, action,
2148                                           0, QB_MIN(timeout_ms, INT_MAX),
2149                                           params, 0);
2150     if (op == NULL) {
2151         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
2152                  action, rsc_class, (rsc_prov? ":" : ""),
2153                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
2154         g_hash_table_destroy(params);
2155         return CRM_EX_OSERR;
2156     }
2157 
2158 #if PCMK__ENABLE_SERVICE
2159     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
2160         class = resources_find_service_class(rsc_type);
2161     }
2162 #endif
2163 
2164     if (!pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_cli_exec)) {
2165         services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
2166                                 "Manual execution of the %s standard is "
2167                                 "unsupported", pcmk__s(class, "unspecified"));
2168     }
2169 
2170     if (op->rc != PCMK_OCF_UNKNOWN) {
2171         exit_code = op->rc;
2172         goto done;
2173     }
2174 
2175     services_action_sync(op);
2176 
2177     // Map results to OCF codes for consistent reporting to user
2178     {
2179         enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
2180 
2181         // Cast variable instead of function return to keep compilers happy
2182         exit_code = (crm_exit_t) ocf_code;
2183     }
2184 
2185 done:
2186     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
2187                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
2188                  exit_code, op->status, services__exit_reason(op),
2189                  op->stdout_data, op->stderr_data);
2190     services_action_free(op);
2191     return exit_code;
2192 }
2193 
2194 /*!
2195  * \internal
2196  * \brief Get the timeout the cluster would use for an action
2197  *
2198  * \param[in] rsc     Resource that action is for
2199  * \param[in] action  Name of action
2200  */
2201 static guint
2202 get_action_timeout(pcmk_resource_t *rsc, const char *action)
     /*  */
2203 {
2204     long long timeout_ms = -1LL;
2205     xmlNode *op = pcmk__find_action_config(rsc, action, 0, true);
2206     GHashTable *meta = pcmk__unpack_action_meta(rsc, NULL, action, 0, op);
2207 
2208     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
2209                        &timeout_ms, -1LL) != pcmk_rc_ok)
2210         || (timeout_ms <= 0LL)) {
2211         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
2212     }
2213     g_hash_table_destroy(meta);
2214     return (guint) QB_MIN(timeout_ms, UINT_MAX);
2215 }
2216 
2217 crm_exit_t
2218 cli_resource_execute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
2219                      const char *rsc_action, GHashTable *override_hash,
2220                      guint timeout_ms, cib_t *cib, pcmk_scheduler_t *scheduler,
2221                      int resource_verbose, gboolean force, int check_level)
2222 {
2223     pcmk__output_t *out = scheduler->priv;
2224     crm_exit_t exit_code = CRM_EX_OK;
2225     const char *rid = NULL;
2226     const char *rtype = NULL;
2227     const char *rprov = NULL;
2228     const char *rclass = NULL;
2229     GHashTable *params = NULL;
2230 
2231     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
2232                                     "force-promote", NULL)) {
2233         if (pcmk__is_clone(rsc)) {
2234             GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
2235             if(nodes != NULL && force == FALSE) {
2236                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
2237                          rsc_action, rsc->id);
2238                 out->err(out,
2239                          "Try setting "
2240                          PCMK_META_TARGET_ROLE "=" PCMK_ROLE_STOPPED
2241                          " first or specifying the force option");
2242                 return CRM_EX_UNSAFE;
2243             }
2244 
2245             g_list_free_full(nodes, free);
2246         }
2247     }
2248 
2249     if (pcmk__is_clone(rsc)) {
2250         /* Grab the first child resource in the hope it's not a group */
2251         rsc = rsc->children->data;
2252     }
2253 
2254     if (pcmk__is_group(rsc)) {
2255         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
2256         return CRM_EX_UNIMPLEMENT_FEATURE;
2257     } else if (pcmk__is_bundled(rsc)) {
2258         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
2259         return CRM_EX_UNIMPLEMENT_FEATURE;
2260     }
2261 
2262     rclass = crm_element_value(rsc->xml, PCMK_XA_CLASS);
2263     rprov = crm_element_value(rsc->xml, PCMK_XA_PROVIDER);
2264     rtype = crm_element_value(rsc->xml, PCMK_XA_TYPE);
2265 
2266     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
2267                                       scheduler);
2268 
2269     if (timeout_ms == 0U) {
2270         timeout_ms = get_action_timeout(rsc, get_action(rsc_action));
2271     }
2272 
2273     rid = pcmk__is_anonymous_clone(rsc->parent)? requested_name : rsc->id;
2274 
2275     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
2276                                                  params, override_hash, timeout_ms,
2277                                                  resource_verbose, force, check_level);
2278     return exit_code;
2279 }
2280 
2281 // \return Standard Pacemaker return code
2282 int
2283 cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
     /*  */
2284                   const char *host_name, const char *move_lifetime, cib_t *cib,
2285                   int cib_options, pcmk_scheduler_t *scheduler,
2286                   gboolean promoted_role_only, gboolean force)
2287 {
2288     pcmk__output_t *out = scheduler->priv;
2289     int rc = pcmk_rc_ok;
2290     unsigned int count = 0;
2291     pcmk_node_t *current = NULL;
2292     pcmk_node_t *dest = pcmk_find_node(scheduler, host_name);
2293     bool cur_is_dest = false;
2294 
2295     if (dest == NULL) {
2296         return pcmk_rc_node_unknown;
2297     }
2298 
2299     if (promoted_role_only
2300         && !pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
2301 
2302         const pcmk_resource_t *p = pe__const_top_resource(rsc, false);
2303 
2304         if (pcmk_is_set(p->flags, pcmk_rsc_promotable)) {
2305             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
2306             rsc_id = p->id;
2307             rsc = p;
2308 
2309         } else {
2310             out->info(out, "Ignoring --promoted option: %s is not promotable",
2311                       rsc_id);
2312             promoted_role_only = FALSE;
2313         }
2314     }
2315 
2316     current = pe__find_active_requires(rsc, &count);
2317 
2318     if (pcmk_is_set(rsc->flags, pcmk_rsc_promotable)) {
2319         unsigned int promoted_count = 0;
2320         pcmk_node_t *promoted_node = NULL;
2321 
2322         for (const GList *iter = rsc->children; iter; iter = iter->next) {
2323             const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
2324             enum rsc_role_e child_role = child->fns->state(child, TRUE);
2325 
2326             if (child_role == pcmk_role_promoted) {
2327                 rsc = child;
2328                 promoted_node = pcmk__current_node(child);
2329                 promoted_count++;
2330             }
2331         }
2332         if (promoted_role_only || (promoted_count != 0)) {
2333             count = promoted_count;
2334             current = promoted_node;
2335         }
2336 
2337     }
2338 
2339     if (count > 1) {
2340         if (pcmk__is_clone(rsc)) {
2341             current = NULL;
2342         } else {
2343             return pcmk_rc_multiple;
2344         }
2345     }
2346 
2347     if (pcmk__same_node(current, dest)) {
2348         cur_is_dest = true;
2349         if (force) {
2350             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
2351                      rsc_id, promoted_role_only?"promoted":"active",
2352                      pcmk__node_name(dest));
2353         } else {
2354             return pcmk_rc_already;
2355         }
2356     }
2357 
2358     /* Clear any previous prefer constraints across all nodes. */
2359     cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, cib_options, false,
2360                        force);
2361 
2362     /* Clear any previous ban constraints on 'dest'. */
2363     cli_resource_clear(rsc_id, dest->details->uname, scheduler->nodes, cib,
2364                        cib_options, TRUE, force);
2365 
2366     /* Record an explicit preference for 'dest' */
2367     rc = cli_resource_prefer(out, rsc_id, dest->details->uname, move_lifetime,
2368                              cib, cib_options, promoted_role_only,
2369                              PCMK_ROLE_PROMOTED);
2370 
2371     crm_trace("%s%s now prefers %s%s",
2372               rsc->id, (promoted_role_only? " (promoted)" : ""),
2373               pcmk__node_name(dest), force?"(forced)":"");
2374 
2375     /* only ban the previous location if current location != destination location.
2376      * it is possible to use -M to enforce a location without regard of where the
2377      * resource is currently located */
2378     if (force && !cur_is_dest) {
2379         /* Ban the original location if possible */
2380         if(current) {
2381             (void)cli_resource_ban(out, rsc_id, current->details->uname, move_lifetime,
2382                                    cib, cib_options, promoted_role_only,
2383                                    PCMK_ROLE_PROMOTED);
2384         } else if(count > 1) {
2385             out->info(out, "Resource '%s' is currently %s in %d locations. "
2386                       "One may now move to %s",
2387                       rsc_id, (promoted_role_only? "promoted" : "active"),
2388                       count, pcmk__node_name(dest));
2389             out->info(out, "To prevent '%s' from being %s at a specific location, "
2390                       "specify a node.",
2391                       rsc_id, (promoted_role_only? "promoted" : "active"));
2392 
2393         } else {
2394             crm_trace("Not banning %s from its current location: not active", rsc_id);
2395         }
2396     }
2397 
2398     return rc;
2399 }