root/tools/crm_resource_runtime.c

/* */

DEFINITIONS

1. build_node_info_list
2. cli_resource_search
3. find_resource_attr
4. find_matching_attr_resources_recursive
5. find_matching_attr_resources
6. get_cib_rsc
7. update_element_attribute
8. resources_with_attr
9. free_attr_update_data
10. update_attribute
11. cli_resource_update_attribute
12. cli_resource_delete_attribute
13. send_lrm_rsc_op
14. rsc_fail_name
15. clear_rsc_history
16. clear_rsc_failures
17. clear_rsc_fail_attrs
18. cli_resource_delete
19. cli_cleanup_all
20. check_role
21. check_managed
22. check_locked
23. node_is_unhealthy
24. check_node_health
25. cli_resource_check
26. cli_resource_fail
27. generate_resource_params
28. resource_is_running_on
29. get_active_resources
30. dump_list
31. display_list
32. update_scheduler_input
33. update_dataset
34. max_rsc_stop_timeout
35. wait_time_estimate
36. cli_resource_restart
37. action_is_pending
38. actions_are_pending
39. print_pending_actions
40. wait_till_stable
41. get_action
42. set_agent_environment
43. apply_overrides
44. cli_resource_execute_from_params
45. get_action_timeout
46. cli_resource_execute
47. cli_resource_move

   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <stdio.h>
  13 #include <limits.h>
  14 #include <glib.h>
  15 #include <libxml/tree.h>
  16 
  17 #include <crm/common/ipc_attrd_internal.h>
  18 #include <crm/common/ipc_controld.h>
  19 #include <crm/common/lists_internal.h>
  20 #include <crm/services_internal.h>
  21 
  22 #include <crm_resource.h>
  23 
  24 static GList *
  25 build_node_info_list(const pcmk_resource_t *rsc)
     /*  */
  26 {
  27     GList *retval = NULL;
  28 
  29     for (const GList *iter = rsc->priv->children;
  30          iter != NULL; iter = iter->next) {
  31 
  32         const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
  33 
  34         for (const GList *iter2 = child->priv->active_nodes;
  35              iter2 != NULL; iter2 = iter2->next) {
  36 
  37             const pcmk_node_t *node = (const pcmk_node_t *) iter2->data;
  38             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
  39 
  40             ni->node_name = node->priv->name;
  41             if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)
  42                 && (child->priv->fns->state(child,
  43                                             TRUE) == pcmk_role_promoted)) {
  44                 ni->promoted = true;
  45             }
  46 
  47             retval = g_list_prepend(retval, ni);
  48         }
  49     }
  50 
  51     return retval;
  52 }
  53 
  54 GList *
  55 cli_resource_search(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
  56                     pcmk_scheduler_t *scheduler)
  57 {
  58     GList *retval = NULL;
  59     const pcmk_resource_t *parent = pe__const_top_resource(rsc, false);
  60 
  61     if (pcmk__is_clone(rsc)) {
  62         retval = build_node_info_list(rsc);
  63 
  64     /* The anonymous clone children's common ID is supplied */
  65     } else if (pcmk__is_clone(parent)
  66                && !pcmk_is_set(rsc->flags, pcmk__rsc_unique)
  67                && (rsc->priv->history_id != NULL)
  68                && pcmk__str_eq(requested_name, rsc->priv->history_id,
  69                                pcmk__str_none)
  70                && !pcmk__str_eq(requested_name, rsc->id, pcmk__str_none)) {
  71 
  72         retval = build_node_info_list(parent);
  73 
  74     } else {
  75         for (GList *iter = rsc->priv->active_nodes;
  76              iter != NULL; iter = iter->next) {
  77 
  78             pcmk_node_t *node = (pcmk_node_t *) iter->data;
  79             node_info_t *ni = pcmk__assert_alloc(1, sizeof(node_info_t));
  80 
  81             ni->node_name = node->priv->name;
  82             if (rsc->priv->fns->state(rsc, TRUE) == pcmk_role_promoted) {
  83                 ni->promoted = true;
  84             }
  85 
  86             retval = g_list_prepend(retval, ni);
  87         }
  88     }
  89 
  90     return retval;
  91 }
  92 
  93 // \return Standard Pacemaker return code
  94 static int
  95 find_resource_attr(pcmk__output_t *out, cib_t * the_cib, const char *attr,
     /*  */
  96                    const char *rsc, const char *attr_set_type, const char *set_name,
  97                    const char *attr_id, const char *attr_name, xmlNode **result)
  98 {
  99     xmlNode *xml_search;
 100     int rc = pcmk_rc_ok;
 101     GString *xpath = NULL;
 102     const char *xpath_base = NULL;
 103 
 104     if (result) {
 105         *result = NULL;
 106     }
 107 
 108     if(the_cib == NULL) {
 109         return ENOTCONN;
 110     }
 111 
 112     xpath_base = pcmk_cib_xpath_for(PCMK_XE_RESOURCES);
 113     if (xpath_base == NULL) {
 114         crm_err(PCMK_XE_RESOURCES " CIB element not known (bug?)");
 115         return ENOMSG;
 116     }
 117 
 118     xpath = g_string_sized_new(1024);
 119     pcmk__g_strcat(xpath,
 120                    xpath_base, "//*[@" PCMK_XA_ID "=\"", rsc, "\"]", NULL);
 121 
 122     if (attr_set_type != NULL) {
 123         pcmk__g_strcat(xpath, "/", attr_set_type, NULL);
 124         if (set_name != NULL) {
 125             pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "=\"", set_name, "\"]",
 126                            NULL);
 127         }
 128     }
 129 
 130     g_string_append(xpath, "//" PCMK_XE_NVPAIR);
 131 
 132     if (attr_id != NULL && attr_name!= NULL) {
 133         pcmk__g_strcat(xpath,
 134                        "[@" PCMK_XA_ID "='", attr_id, "' "
 135                        "and @" PCMK_XA_NAME "='", attr_name, "']", NULL);
 136 
 137     } else if (attr_id != NULL) {
 138         pcmk__g_strcat(xpath, "[@" PCMK_XA_ID "='", attr_id, "']", NULL);
 139 
 140     } else if (attr_name != NULL) {
 141         pcmk__g_strcat(xpath, "[@" PCMK_XA_NAME "='", attr_name, "']", NULL);
 142     }
 143 
 144     rc = the_cib->cmds->query(the_cib, xpath->str, &xml_search,
 145                               cib_sync_call|cib_xpath);
 146     rc = pcmk_legacy2rc(rc);
 147 
 148     if (rc == pcmk_rc_ok) {
 149         crm_log_xml_debug(xml_search, "Match");
 150         if (xml_search->children != NULL) {
 151             rc = ENOTUNIQ;
 152             pcmk__warn_multiple_name_matches(out, xml_search, attr_name);
 153             out->spacer(out);
 154         }
 155     }
 156 
 157     if (result) {
 158         *result = xml_search;
 159     } else {
 160         pcmk__xml_free(xml_search);
 161     }
 162 
 163     g_string_free(xpath, TRUE);
 164     return rc;
 165 }
 166 
 167 /* PRIVATE. Use the find_matching_attr_resources instead. */
 168 static void
 169 find_matching_attr_resources_recursive(pcmk__output_t *out,
     /*  */
 170                                        GList /* <pcmk_resource_t*> */ **result,
 171                                        pcmk_resource_t *rsc, const char * attr_set,
 172                                        const char * attr_set_type, const char * attr_id,
 173                                        const char * attr_name, cib_t * cib, int depth)
 174 {
 175     int rc = pcmk_rc_ok;
 176     char *lookup_id = clone_strip(rsc->id);
 177 
 178     for (GList *gIter = rsc->priv->children;
 179          gIter != NULL; gIter = gIter->next) {
 180 
 181         find_matching_attr_resources_recursive(out, result,
 182                                                (pcmk_resource_t *) gIter->data,
 183                                                attr_set, attr_set_type, attr_id,
 184                                                attr_name, cib, depth+1);
 185         /* do it only once for clones */
 186         if (pcmk__is_clone(rsc)) {
 187             break;
 188         }
 189     }
 190 
 191     rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 192                             attr_set, attr_id, attr_name, NULL);
 193     /* Post-order traversal.
 194      * The root is always on the list and it is the last item. */
 195     if((0 == depth) || (pcmk_rc_ok == rc)) {
 196         /* push the head */
 197         *result = g_list_append(*result, rsc);
 198     }
 199 
 200     free(lookup_id);
 201 }
 202 
 203 
 204 /* The result is a linearized pre-ordered tree of resources. */
 205 static GList/*<pcmk_resource_t*>*/ *
 206 find_matching_attr_resources(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
 207                              const char * rsc_id, const char * attr_set,
 208                              const char * attr_set_type, const char * attr_id,
 209                              const char * attr_name, cib_t * cib, const char * cmd,
 210                              gboolean force)
 211 {
 212     int rc = pcmk_rc_ok;
 213     char *lookup_id = NULL;
 214     GList * result = NULL;
 215 
 216     /* If --force is used, update only the requested resource (clone or primitive).
 217      * Otherwise, if the primitive has the attribute, use that.
 218      * Otherwise use the clone. */
 219     if(force == TRUE) {
 220         return g_list_append(result, rsc);
 221     }
 222     if (pcmk__is_clone(rsc->priv->parent)) {
 223         int rc = find_resource_attr(out, cib, PCMK_XA_ID, rsc_id, attr_set_type,
 224                                     attr_set, attr_id, attr_name, NULL);
 225 
 226         if(rc != pcmk_rc_ok) {
 227             rsc = rsc->priv->parent;
 228             out->info(out, "Performing %s of '%s' on '%s', the parent of '%s'",
 229                       cmd, attr_name, rsc->id, rsc_id);
 230         }
 231         return g_list_append(result, rsc);
 232 
 233     } else if ((rsc->priv->parent == NULL)
 234                && (rsc->priv->children != NULL) && pcmk__is_clone(rsc)) {
 235 
 236         pcmk_resource_t *child = rsc->priv->children->data;
 237 
 238         if (pcmk__is_primitive(child)) {
 239             lookup_id = clone_strip(child->id); /* Could be a cloned group! */
 240             rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id,
 241                                     attr_set_type, attr_set, attr_id, attr_name, NULL);
 242 
 243             if(rc == pcmk_rc_ok) {
 244                 rsc = child;
 245                 out->info(out, "A value for '%s' already exists in child '%s', performing %s on that instead of '%s'",
 246                           attr_name, lookup_id, cmd, rsc_id);
 247             }
 248 
 249             free(lookup_id);
 250         }
 251         return g_list_append(result, rsc);
 252     }
 253     /* If the resource is a group ==> children inherit the attribute if defined. */
 254     find_matching_attr_resources_recursive(out, &result, rsc, attr_set,
 255                                            attr_set_type, attr_id, attr_name,
 256                                            cib, 0);
 257     return result;
 258 }
 259 
 260 /*!
 261  * \internal
 262  * \brief Get a resource's XML by resource ID from a given CIB XML tree
 263  *
 264  * \param[in] cib_xml  CIB XML to search
 265  * \param[in] rsc      Resource whose XML to get
 266  *
 267  * \return Subtree of \p cib_xml belonging to \p rsc, or \c NULL if not found
 268  */
 269 static xmlNode *
 270 get_cib_rsc(xmlNode *cib_xml, const pcmk_resource_t *rsc)
     /*  */
 271 {
 272     char *xpath = crm_strdup_printf("%s//*[@" PCMK_XA_ID "='%s']",
 273                                     pcmk_cib_xpath_for(PCMK_XE_RESOURCES),
 274                                     pcmk__xe_id(rsc->priv->xml));
 275     xmlNode *rsc_xml = get_xpath_object(xpath, cib_xml, LOG_ERR);
 276 
 277     free(xpath);
 278     return rsc_xml;
 279 }
 280 
 281 static int
 282 update_element_attribute(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
 283                          cib_t *cib, xmlNode *cib_xml_orig,
 284                          const char *attr_name, const char *attr_value)
 285 {
 286     int rc = pcmk_rc_ok;
 287     xmlNode *rsc_xml = rsc->priv->xml;
 288 
 289     rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
 290     if (rsc_xml == NULL) {
 291         return ENXIO;
 292     }
 293 
 294     crm_xml_add(rsc_xml, attr_name, attr_value);
 295 
 296     rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc_xml, cib_sync_call);
 297     rc = pcmk_legacy2rc(rc);
 298     if (rc == pcmk_rc_ok) {
 299         out->info(out, "Set attribute: " PCMK_XA_NAME "=%s value=%s",
 300                   attr_name, attr_value);
 301     }
 302 
 303     return rc;
 304 }
 305 
 306 static int
 307 resources_with_attr(pcmk__output_t *out, cib_t *cib, pcmk_resource_t *rsc,
     /*  */
 308                     const char *requested_name, const char *attr_set,
 309                     const char *attr_set_type, const char *attr_id,
 310                     const char *attr_name, const char *top_id, gboolean force,
 311                     GList **resources)
 312 {
 313     if (pcmk__str_eq(attr_set_type, PCMK_XE_INSTANCE_ATTRIBUTES,
 314                      pcmk__str_casei)) {
 315         if (!force) {
 316             xmlNode *xml_search = NULL;
 317             int rc = pcmk_rc_ok;
 318 
 319             rc = find_resource_attr(out, cib, PCMK_XA_ID, top_id,
 320                                     PCMK_XE_META_ATTRIBUTES, attr_set, attr_id,
 321                                     attr_name, &xml_search);
 322 
 323             if (rc == pcmk_rc_ok || rc == ENOTUNIQ) {
 324                 char *found_attr_id = NULL;
 325 
 326                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 327 
 328                 if (!out->is_quiet(out)) {
 329                     out->err(out,
 330                              "WARNING: There is already a meta attribute "
 331                              "for '%s' called '%s' (id=%s)",
 332                              top_id, attr_name, found_attr_id);
 333                     out->err(out,
 334                              "         Delete '%s' first or use the force option "
 335                              "to override", found_attr_id);
 336                 }
 337 
 338                 free(found_attr_id);
 339                 pcmk__xml_free(xml_search);
 340                 return ENOTUNIQ;
 341             }
 342 
 343             pcmk__xml_free(xml_search);
 344         }
 345 
 346         *resources = g_list_append(*resources, rsc);
 347 
 348     } else {
 349         *resources = find_matching_attr_resources(out, rsc, requested_name,
 350                                                   attr_set, attr_set_type,
 351                                                   attr_id, attr_name, cib,
 352                                                   "update", force);
 353     }
 354 
 355     /* If the user specified attr_set or attr_id, the intent is to modify a
 356      * single resource, which will be the last item in the list.
 357      */
 358     if ((attr_set != NULL) || (attr_id != NULL)) {
 359         GList *last = g_list_last(*resources);
 360 
 361         *resources = g_list_remove_link(*resources, last);
 362         g_list_free(*resources);
 363         *resources = last;
 364     }
 365 
 366     return pcmk_rc_ok;
 367 }
 368 
 369 static void
 370 free_attr_update_data(gpointer data)
     /*  */
 371 {
 372     attr_update_data_t *ud = data;
 373 
 374     if (ud == NULL) {
 375         return;
 376     }
 377 
 378     free(ud->attr_set_type);
 379     free(ud->attr_set_id);
 380     free(ud->attr_name);
 381     free(ud->attr_value);
 382     free(ud->given_rsc_id);
 383     free(ud->found_attr_id);
 384     free(ud);
 385 }
 386 
 387 static int
 388 update_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 389                  const char *attr_set, const char *attr_set_type,
 390                  const char *attr_id, const char *attr_name,
 391                  const char *attr_value, gboolean recursive, cib_t *cib,
 392                  xmlNode *cib_xml_orig, gboolean force, GList **results)
 393 {
 394     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
 395     int rc = pcmk_rc_ok;
 396 
 397     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 398     const char *top_id = pe__const_top_resource(rsc, false)->id;
 399 
 400     if ((attr_id == NULL) && !force) {
 401         find_resource_attr(out, cib, PCMK_XA_ID, top_id, NULL, NULL, NULL,
 402                            attr_name, NULL);
 403     }
 404 
 405     rc = resources_with_attr(out, cib, rsc, requested_name, attr_set, attr_set_type,
 406                              attr_id, attr_name, top_id, force, &resources);
 407 
 408     if (rc != pcmk_rc_ok) {
 409         return rc;
 410     }
 411 
 412     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 413         // @TODO Functionize loop body to simplify freeing allocated memory
 414         char *lookup_id = NULL;
 415         char *local_attr_set = NULL;
 416         char *found_attr_id = NULL;
 417         const char *rsc_attr_id = attr_id;
 418         const char *rsc_attr_set = attr_set;
 419 
 420         xmlNode *rsc_xml = rsc->priv->xml;
 421         xmlNode *xml_top = NULL;
 422         xmlNode *xml_obj = NULL;
 423         xmlNode *xml_search = NULL;
 424 
 425         rsc = (pcmk_resource_t *) iter->data;
 426 
 427         lookup_id = clone_strip(rsc->id); /* Could be a cloned group! */
 428         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 429                                 attr_set, attr_id, attr_name, &xml_search);
 430 
 431         switch (rc) {
 432             case pcmk_rc_ok:
 433                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 434                 crm_debug("Found a match for " PCMK_XA_NAME "='%s': "
 435                           PCMK_XA_ID "='%s'", attr_name, found_attr_id);
 436                 rsc_attr_id = found_attr_id;
 437                 break;
 438 
 439             case ENXIO:
 440                 if (rsc_attr_set == NULL) {
 441                     local_attr_set = crm_strdup_printf("%s-%s", lookup_id,
 442                                                        attr_set_type);
 443                     rsc_attr_set = local_attr_set;
 444                 }
 445                 if (rsc_attr_id == NULL) {
 446                     found_attr_id = crm_strdup_printf("%s-%s",
 447                                                       rsc_attr_set, attr_name);
 448                     rsc_attr_id = found_attr_id;
 449                 }
 450 
 451                 rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
 452                 if (rsc_xml == NULL) {
 453                     /* @TODO Warn and continue through the rest of the resources
 454                      * and return the error at the end? This should never
 455                      * happen, but if it does, then we could have a partial
 456                      * update.
 457                      */
 458                     free(lookup_id);
 459                     free(found_attr_id);
 460                     pcmk__xml_free(xml_search);
 461                     g_list_free(resources);
 462                     return ENXIO;
 463                 }
 464 
 465                 xml_top = pcmk__xe_create(NULL, (const char *) rsc_xml->name);
 466                 crm_xml_add(xml_top, PCMK_XA_ID, lookup_id);
 467 
 468                 xml_obj = pcmk__xe_create(xml_top, attr_set_type);
 469                 crm_xml_add(xml_obj, PCMK_XA_ID, rsc_attr_set);
 470                 break;
 471 
 472             default:
 473                 free(lookup_id);
 474                 free(found_attr_id);
 475                 pcmk__xml_free(xml_search);
 476                 g_list_free(resources);
 477                 return rc;
 478         }
 479 
 480         xml_obj = crm_create_nvpair_xml(xml_obj, rsc_attr_id, attr_name,
 481                                         attr_value);
 482         if (xml_top == NULL) {
 483             xml_top = xml_obj;
 484         }
 485 
 486         crm_log_xml_debug(xml_top, "Update");
 487 
 488         rc = cib->cmds->modify(cib, PCMK_XE_RESOURCES, xml_top, cib_sync_call);
 489         rc = pcmk_legacy2rc(rc);
 490         if (rc == pcmk_rc_ok) {
 491             attr_update_data_t *ud = pcmk__assert_alloc(1, sizeof(attr_update_data_t));
 492 
 493             if (attr_set_type == NULL) {
 494                 attr_set_type = (const char *) xml_search->parent->name;
 495             }
 496 
 497             if (rsc_attr_set == NULL) {
 498                 rsc_attr_set = crm_element_value(xml_search->parent, PCMK_XA_ID);
 499             }
 500 
 501             ud->attr_set_type = pcmk__str_copy(attr_set_type);
 502             ud->attr_set_id = pcmk__str_copy(rsc_attr_set);
 503             ud->attr_name = pcmk__str_copy(attr_name);
 504             ud->attr_value = pcmk__str_copy(attr_value);
 505             ud->given_rsc_id = pcmk__str_copy(lookup_id);
 506             ud->found_attr_id = pcmk__str_copy(found_attr_id);
 507             ud->rsc = rsc;
 508 
 509             *results = g_list_append(*results, ud);
 510         }
 511 
 512         pcmk__xml_free(xml_top);
 513         pcmk__xml_free(xml_search);
 514 
 515         free(lookup_id);
 516         free(found_attr_id);
 517         free(local_attr_set);
 518 
 519         if (recursive
 520             && pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES,
 521                             pcmk__str_casei)) {
 522             /* We want to set the attribute only on resources explicitly
 523              * colocated with this one, so we use
 524              * rsc->priv->with_this_colocations directly rather than the
 525              * with_this_colocations() method.
 526              */
 527             pcmk__set_rsc_flags(rsc, pcmk__rsc_detect_loop);
 528             for (GList *lpc = rsc->priv->with_this_colocations;
 529                  lpc != NULL; lpc = lpc->next) {
 530                 pcmk__colocation_t *cons = (pcmk__colocation_t *) lpc->data;
 531 
 532                 crm_debug("Checking %s %d", cons->id, cons->score);
 533 
 534                 if (pcmk_is_set(cons->dependent->flags, pcmk__rsc_detect_loop)
 535                     || (cons->score <= 0)) {
 536                     continue;
 537                 }
 538 
 539                 crm_debug("Setting %s=%s for dependent resource %s",
 540                           attr_name, attr_value, cons->dependent->id);
 541                 update_attribute(cons->dependent, cons->dependent->id, NULL,
 542                                  attr_set_type, NULL, attr_name, attr_value,
 543                                  recursive, cib, cib_xml_orig, force, results);
 544             }
 545         }
 546     }
 547 
 548     g_list_free(resources);
 549     return rc;
 550 }
 551 
 552 // \return Standard Pacemaker return code
 553 int
 554 cli_resource_update_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 555                               const char *attr_set, const char *attr_set_type,
 556                               const char *attr_id, const char *attr_name,
 557                               const char *attr_value, gboolean recursive,
 558                               cib_t *cib, xmlNode *cib_xml_orig, gboolean force)
 559 {
 560     static bool need_init = true;
 561     int rc = pcmk_rc_ok;
 562 
 563     GList *results = NULL;
 564     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
 565 
 566     pcmk__assert(cib_xml_orig != NULL);
 567 
 568     /* If we were asked to update the attribute in a resource element (for
 569      * instance, <primitive class="ocf">) there's really not much we need to do.
 570      */
 571     if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 572         return update_element_attribute(out, rsc, cib, cib_xml_orig, attr_name,
 573                                         attr_value);
 574     }
 575 
 576     /* One time initialization - clear flags so we can detect loops */
 577     if (need_init) {
 578         need_init = false;
 579         pcmk__unpack_constraints(rsc->priv->scheduler);
 580         pe__clear_resource_flags_on_all(rsc->priv->scheduler,
 581                                         pcmk__rsc_detect_loop);
 582     }
 583 
 584     rc = update_attribute(rsc, requested_name, attr_set, attr_set_type,
 585                           attr_id, attr_name, attr_value, recursive, cib,
 586                           cib_xml_orig, force, &results);
 587 
 588     if (rc == pcmk_rc_ok) {
 589         if (results == NULL) {
 590             return rc;
 591         }
 592 
 593         out->message(out, "attribute-changed-list", results);
 594         g_list_free_full(results, free_attr_update_data);
 595     }
 596 
 597     return rc;
 598 }
 599 
 600 // \return Standard Pacemaker return code
 601 int
 602 cli_resource_delete_attribute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
 603                               const char *attr_set, const char *attr_set_type,
 604                               const char *attr_id, const char *attr_name,
 605                               cib_t *cib, xmlNode *cib_xml_orig, gboolean force)
 606 {
 607     pcmk__output_t *out = rsc->priv->scheduler->priv->out;
 608     int rc = pcmk_rc_ok;
 609     GList/*<pcmk_resource_t*>*/ *resources = NULL;
 610 
 611     pcmk__assert((cib != NULL) && (cib_xml_orig != NULL));
 612 
 613     if ((attr_id == NULL) && !force) {
 614         find_resource_attr(out, cib, PCMK_XA_ID,
 615                            pe__const_top_resource(rsc, false)->id, NULL,
 616                            NULL, NULL, attr_name, NULL);
 617     }
 618 
 619     if (pcmk__str_eq(attr_set_type, ATTR_SET_ELEMENT, pcmk__str_none)) {
 620         xmlNode *rsc_xml = rsc->priv->xml;
 621 
 622         rsc_xml = get_cib_rsc(cib_xml_orig, rsc);
 623         if (rsc_xml == NULL) {
 624             return ENXIO;
 625         }
 626 
 627         pcmk__xe_remove_attr(rsc_xml, attr_name);
 628         rc = cib->cmds->replace(cib, PCMK_XE_RESOURCES, rsc_xml, cib_sync_call);
 629         rc = pcmk_legacy2rc(rc);
 630         if (rc == pcmk_rc_ok) {
 631             out->info(out, "Deleted attribute: %s", attr_name);
 632         }
 633         return rc;
 634     }
 635 
 636     if (pcmk__str_eq(attr_set_type, PCMK_XE_META_ATTRIBUTES, pcmk__str_none)) {
 637         resources = find_matching_attr_resources(out, rsc, requested_name,
 638                                                  attr_set, attr_set_type,
 639                                                  attr_id, attr_name, cib,
 640                                                  "delete", force);
 641     } else {
 642         resources = g_list_append(resources, rsc);
 643     }
 644 
 645     for (GList *iter = resources; iter != NULL; iter = iter->next) {
 646         char *lookup_id = NULL;
 647         xmlNode *xml_obj = NULL;
 648         xmlNode *xml_search = NULL;
 649         char *found_attr_id = NULL;
 650         const char *rsc_attr_id = attr_id;
 651 
 652         rsc = (pcmk_resource_t *) iter->data;
 653 
 654         /* @TODO Search the original CIB in find_resource_attr() for
 655          * future-proofing, to ensure that we're getting IDs of nvpairs that
 656          * exist in the CIB.
 657          */
 658         lookup_id = clone_strip(rsc->id);
 659         rc = find_resource_attr(out, cib, PCMK_XA_ID, lookup_id, attr_set_type,
 660                                 attr_set, attr_id, attr_name, &xml_search);
 661         switch (rc) {
 662             case pcmk_rc_ok:
 663                 found_attr_id = crm_element_value_copy(xml_search, PCMK_XA_ID);
 664                 pcmk__xml_free(xml_search);
 665                 break;
 666 
 667             case ENXIO:
 668                 free(lookup_id);
 669                 pcmk__xml_free(xml_search);
 670                 continue;
 671 
 672             default:
 673                 free(lookup_id);
 674                 pcmk__xml_free(xml_search);
 675                 g_list_free(resources);
 676                 return rc;
 677         }
 678 
 679         if (rsc_attr_id == NULL) {
 680             rsc_attr_id = found_attr_id;
 681         }
 682 
 683         xml_obj = crm_create_nvpair_xml(NULL, rsc_attr_id, attr_name, NULL);
 684         crm_log_xml_debug(xml_obj, "Delete");
 685 
 686         rc = cib->cmds->remove(cib, PCMK_XE_RESOURCES, xml_obj, cib_sync_call);
 687         rc = pcmk_legacy2rc(rc);
 688 
 689         if (rc == pcmk_rc_ok) {
 690             out->info(out, "Deleted '%s' option: " PCMK_XA_ID "=%s%s%s%s%s",
 691                       lookup_id, found_attr_id,
 692                       ((attr_set == NULL)? "" : " set="),
 693                       pcmk__s(attr_set, ""),
 694                       ((attr_name == NULL)? "" : " " PCMK_XA_NAME "="),
 695                       pcmk__s(attr_name, ""));
 696         }
 697 
 698         free(lookup_id);
 699         pcmk__xml_free(xml_obj);
 700         free(found_attr_id);
 701     }
 702 
 703     g_list_free(resources);
 704     return rc;
 705 }
 706 
 707 // \return Standard Pacemaker return code
 708 static int
 709 send_lrm_rsc_op(pcmk_ipc_api_t *controld_api, bool do_fail_resource,
     /*  */
 710                 const char *host_uname, const char *rsc_id,
 711                 pcmk_scheduler_t *scheduler)
 712 {
 713     pcmk__output_t *out = scheduler->priv->out;
 714     const char *router_node = host_uname;
 715     const char *rsc_api_id = NULL;
 716     const char *rsc_long_id = NULL;
 717     const char *rsc_class = NULL;
 718     const char *rsc_provider = NULL;
 719     const char *rsc_type = NULL;
 720     bool cib_only = false;
 721     pcmk_resource_t *rsc = pe_find_resource(scheduler->priv->resources, rsc_id);
 722 
 723     if (rsc == NULL) {
 724         out->err(out, "Resource %s not found", rsc_id);
 725         return ENXIO;
 726 
 727     } else if (!pcmk__is_primitive(rsc)) {
 728         out->err(out, "We can only process primitive resources, not %s", rsc_id);
 729         return EINVAL;
 730     }
 731 
 732     rsc_class = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
 733     rsc_provider = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER);
 734     rsc_type = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
 735     if ((rsc_class == NULL) || (rsc_type == NULL)) {
 736         out->err(out, "Resource %s does not have a class and type", rsc_id);
 737         return EINVAL;
 738     }
 739 
 740     {
 741         pcmk_node_t *node = pcmk_find_node(scheduler, host_uname);
 742 
 743         if (node == NULL) {
 744             out->err(out, "Node %s not found", host_uname);
 745             return pcmk_rc_node_unknown;
 746         }
 747 
 748         if (!(node->details->online)) {
 749             if (do_fail_resource) {
 750                 out->err(out, "Node %s is not online", host_uname);
 751                 return ENOTCONN;
 752             } else {
 753                 cib_only = true;
 754             }
 755         }
 756         if (!cib_only && pcmk__is_pacemaker_remote_node(node)) {
 757             node = pcmk__current_node(node->priv->remote);
 758             if (node == NULL) {
 759                 out->err(out, "No cluster connection to Pacemaker Remote node %s detected",
 760                          host_uname);
 761                 return ENOTCONN;
 762             }
 763             router_node = node->priv->name;
 764         }
 765     }
 766 
 767     if (rsc->priv->history_id != NULL) {
 768         rsc_api_id = rsc->priv->history_id;
 769         rsc_long_id = rsc->id;
 770     } else {
 771         rsc_api_id = rsc->id;
 772     }
 773     if (do_fail_resource) {
 774         return pcmk_controld_api_fail(controld_api, host_uname, router_node,
 775                                       rsc_api_id, rsc_long_id,
 776                                       rsc_class, rsc_provider, rsc_type);
 777     } else {
 778         return pcmk_controld_api_refresh(controld_api, host_uname, router_node,
 779                                          rsc_api_id, rsc_long_id, rsc_class,
 780                                          rsc_provider, rsc_type, cib_only);
 781     }
 782 }
 783 
 784 /*!
 785  * \internal
 786  * \brief Get resource name as used in failure-related node attributes
 787  *
 788  * \param[in] rsc  Resource to check
 789  *
 790  * \return Newly allocated string containing resource's fail name
 791  * \note The caller is responsible for freeing the result.
 792  */
 793 static inline char *
 794 rsc_fail_name(const pcmk_resource_t *rsc)
     /*  */
 795 {
 796     const char *name = pcmk__s(rsc->priv->history_id, rsc->id);
 797 
 798     if (pcmk_is_set(rsc->flags, pcmk__rsc_unique)) {
 799         return strdup(name);
 800     }
 801     return clone_strip(name);
 802 }
 803 
 804 // \return Standard Pacemaker return code
 805 static int
 806 clear_rsc_history(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 807                   const char *rsc_id, pcmk_scheduler_t *scheduler)
 808 {
 809     int rc = pcmk_rc_ok;
 810 
 811     /* Erase the resource's entire LRM history in the CIB, even if we're only
 812      * clearing a single operation's fail count. If we erased only entries for a
 813      * single operation, we might wind up with a wrong idea of the current
 814      * resource state, and we might not re-probe the resource.
 815      */
 816     rc = send_lrm_rsc_op(controld_api, false, host_uname, rsc_id, scheduler);
 817     if (rc != pcmk_rc_ok) {
 818         return rc;
 819     }
 820 
 821     crm_trace("Processing %d mainloop inputs",
 822               pcmk_controld_api_replies_expected(controld_api));
 823     while (g_main_context_iteration(NULL, FALSE)) {
 824         crm_trace("Processed mainloop input, %d still remaining",
 825                   pcmk_controld_api_replies_expected(controld_api));
 826     }
 827     return rc;
 828 }
 829 
 830 // \return Standard Pacemaker return code
 831 static int
 832 clear_rsc_failures(pcmk__output_t *out, pcmk_ipc_api_t *controld_api,
     /*  */
 833                    const char *node_name, const char *rsc_id, const char *operation,
 834                    const char *interval_spec, pcmk_scheduler_t *scheduler)
 835 {
 836     int rc = pcmk_rc_ok;
 837     const char *failed_value = NULL;
 838     const char *failed_id = NULL;
 839     char *interval_ms_s = NULL;
 840     GHashTable *rscs = NULL;
 841     GHashTableIter iter;
 842 
 843     /* Create a hash table to use as a set of resources to clean. This lets us
 844      * clean each resource only once (per node) regardless of how many failed
 845      * operations it has.
 846      */
 847     rscs = pcmk__strkey_table(NULL, NULL);
 848 
 849     // Normalize interval to milliseconds for comparison to history entry
 850     if (operation) {
 851         guint interval_ms = 0U;
 852 
 853         pcmk_parse_interval_spec(interval_spec, &interval_ms);
 854         interval_ms_s = crm_strdup_printf("%u", interval_ms);
 855     }
 856 
 857     for (xmlNode *xml_op = pcmk__xe_first_child(scheduler->priv->failed, NULL,
 858                                                 NULL, NULL);
 859          xml_op != NULL; xml_op = pcmk__xe_next(xml_op, NULL)) {
 860 
 861         failed_id = crm_element_value(xml_op, PCMK__XA_RSC_ID);
 862         if (failed_id == NULL) {
 863             // Malformed history entry, should never happen
 864             continue;
 865         }
 866 
 867         // No resource specified means all resources match
 868         if (rsc_id) {
 869             pcmk_resource_t *fail_rsc = NULL;
 870 
 871             fail_rsc = pe_find_resource_with_flags(scheduler->priv->resources,
 872                                                    failed_id,
 873                                                    pcmk_rsc_match_history
 874                                                    |pcmk_rsc_match_anon_basename);
 875             if ((fail_rsc == NULL)
 876                 || !pcmk__str_eq(rsc_id, fail_rsc->id, pcmk__str_none)) {
 877                 continue;
 878             }
 879         }
 880 
 881         // Host name should always have been provided by this point
 882         failed_value = crm_element_value(xml_op, PCMK_XA_UNAME);
 883         if (!pcmk__str_eq(node_name, failed_value, pcmk__str_casei)) {
 884             continue;
 885         }
 886 
 887         // No operation specified means all operations match
 888         if (operation) {
 889             failed_value = crm_element_value(xml_op, PCMK_XA_OPERATION);
 890             if (!pcmk__str_eq(operation, failed_value, pcmk__str_casei)) {
 891                 continue;
 892             }
 893 
 894             // Interval (if operation was specified) defaults to 0 (not all)
 895             failed_value = crm_element_value(xml_op, PCMK_META_INTERVAL);
 896             if (!pcmk__str_eq(interval_ms_s, failed_value, pcmk__str_casei)) {
 897                 continue;
 898             }
 899         }
 900 
 901         g_hash_table_add(rscs, (gpointer) failed_id);
 902     }
 903 
 904     free(interval_ms_s);
 905 
 906     g_hash_table_iter_init(&iter, rscs);
 907     while (g_hash_table_iter_next(&iter, (gpointer *) &failed_id, NULL)) {
 908         crm_debug("Erasing failures of %s on %s", failed_id, node_name);
 909         rc = clear_rsc_history(controld_api, node_name, failed_id, scheduler);
 910         if (rc != pcmk_rc_ok) {
 911             return rc;
 912         }
 913     }
 914     g_hash_table_destroy(rscs);
 915     return rc;
 916 }
 917 
 918 // \return Standard Pacemaker return code
 919 static int
 920 clear_rsc_fail_attrs(const pcmk_resource_t *rsc, const char *operation,
     /*  */
 921                      const char *interval_spec, const pcmk_node_t *node)
 922 {
 923     int rc = pcmk_rc_ok;
 924     int attr_options = pcmk__node_attr_none;
 925     char *rsc_name = rsc_fail_name(rsc);
 926 
 927     if (pcmk__is_pacemaker_remote_node(node)) {
 928         attr_options |= pcmk__node_attr_remote;
 929     }
 930 
 931     rc = pcmk__attrd_api_clear_failures(NULL, node->priv->name, rsc_name,
 932                                         operation, interval_spec, NULL,
 933                                         attr_options);
 934     free(rsc_name);
 935     return rc;
 936 }
 937 
 938 // \return Standard Pacemaker return code
 939 int
 940 cli_resource_delete(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
 941                     const pcmk_resource_t *rsc, const char *operation,
 942                     const char *interval_spec, bool just_failures,
 943                     pcmk_scheduler_t *scheduler, gboolean force)
 944 {
 945     pcmk__output_t *out = scheduler->priv->out;
 946     int rc = pcmk_rc_ok;
 947     pcmk_node_t *node = NULL;
 948 
 949     if (rsc == NULL) {
 950         return ENXIO;
 951 
 952     } else if (rsc->priv->children != NULL) {
 953 
 954         for (const GList *lpc = rsc->priv->children;
 955              lpc != NULL; lpc = lpc->next) {
 956 
 957             const pcmk_resource_t *child = (const pcmk_resource_t *) lpc->data;
 958 
 959             rc = cli_resource_delete(controld_api, host_uname, child, operation,
 960                                      interval_spec, just_failures, scheduler,
 961                                      force);
 962             if (rc != pcmk_rc_ok) {
 963                 return rc;
 964             }
 965         }
 966         return pcmk_rc_ok;
 967 
 968     } else if (host_uname == NULL) {
 969         GList *lpc = NULL;
 970         GList *nodes = g_hash_table_get_values(rsc->priv->probed_nodes);
 971 
 972         if(nodes == NULL && force) {
 973             nodes = pcmk__copy_node_list(scheduler->nodes, false);
 974 
 975         } else if ((nodes == NULL)
 976                    && pcmk_is_set(rsc->flags, pcmk__rsc_exclusive_probes)) {
 977             GHashTableIter iter;
 978             pcmk_node_t *node = NULL;
 979 
 980             g_hash_table_iter_init(&iter, rsc->priv->allowed_nodes);
 981             while (g_hash_table_iter_next(&iter, NULL, (void**)&node)) {
 982                 if (node->assign->score >= 0) {
 983                     nodes = g_list_prepend(nodes, node);
 984                 }
 985             }
 986 
 987         } else if(nodes == NULL) {
 988             nodes = g_hash_table_get_values(rsc->priv->allowed_nodes);
 989         }
 990 
 991         for (lpc = nodes; lpc != NULL; lpc = lpc->next) {
 992             node = (pcmk_node_t *) lpc->data;
 993 
 994             if (node->details->online) {
 995                 rc = cli_resource_delete(controld_api, node->priv->name, rsc,
 996                                          operation, interval_spec, just_failures,
 997                                          scheduler, force);
 998             }
 999             if (rc != pcmk_rc_ok) {
1000                 g_list_free(nodes);
1001                 return rc;
1002             }
1003         }
1004 
1005         g_list_free(nodes);
1006         return pcmk_rc_ok;
1007     }
1008 
1009     node = pcmk_find_node(scheduler, host_uname);
1010 
1011     if (node == NULL) {
1012         out->err(out, "Unable to clean up %s because node %s not found",
1013                  rsc->id, host_uname);
1014         return ENODEV;
1015     }
1016 
1017     if (!pcmk_is_set(node->priv->flags, pcmk__node_probes_allowed)) {
1018         out->err(out, "Unable to clean up %s because resource discovery disabled on %s",
1019                  rsc->id, host_uname);
1020         return EOPNOTSUPP;
1021     }
1022 
1023     if (controld_api == NULL) {
1024         out->err(out, "Dry run: skipping clean-up of %s on %s due to CIB_file",
1025                  rsc->id, host_uname);
1026         return pcmk_rc_ok;
1027     }
1028 
1029     rc = clear_rsc_fail_attrs(rsc, operation, interval_spec, node);
1030     if (rc != pcmk_rc_ok) {
1031         out->err(out, "Unable to clean up %s failures on %s: %s",
1032                  rsc->id, host_uname, pcmk_rc_str(rc));
1033         return rc;
1034     }
1035 
1036     if (just_failures) {
1037         rc = clear_rsc_failures(out, controld_api, host_uname, rsc->id, operation,
1038                                 interval_spec, scheduler);
1039     } else {
1040         rc = clear_rsc_history(controld_api, host_uname, rsc->id, scheduler);
1041     }
1042     if (rc != pcmk_rc_ok) {
1043         out->err(out, "Cleaned %s failures on %s, but unable to clean history: %s",
1044                  rsc->id, host_uname, pcmk_rc_str(rc));
1045     } else {
1046         out->info(out, "Cleaned up %s on %s", rsc->id, host_uname);
1047     }
1048     return rc;
1049 }
1050 
1051 // \return Standard Pacemaker return code
1052 int
1053 cli_cleanup_all(pcmk_ipc_api_t *controld_api, const char *node_name,
     /*  */
1054                 const char *operation, const char *interval_spec,
1055                 pcmk_scheduler_t *scheduler)
1056 {
1057     pcmk__output_t *out = scheduler->priv->out;
1058     int rc = pcmk_rc_ok;
1059     int attr_options = pcmk__node_attr_none;
1060     const char *display_name = node_name? node_name : "all nodes";
1061 
1062     if (controld_api == NULL) {
1063         out->info(out, "Dry run: skipping clean-up of %s due to CIB_file",
1064                   display_name);
1065         return rc;
1066     }
1067 
1068     if (node_name) {
1069         pcmk_node_t *node = pcmk_find_node(scheduler, node_name);
1070 
1071         if (node == NULL) {
1072             out->err(out, "Unknown node: %s", node_name);
1073             return ENXIO;
1074         }
1075         if (pcmk__is_pacemaker_remote_node(node)) {
1076             attr_options |= pcmk__node_attr_remote;
1077         }
1078     }
1079 
1080     rc = pcmk__attrd_api_clear_failures(NULL, node_name, NULL, operation,
1081                                         interval_spec, NULL, attr_options);
1082     if (rc != pcmk_rc_ok) {
1083         out->err(out, "Unable to clean up all failures on %s: %s",
1084                  display_name, pcmk_rc_str(rc));
1085         return rc;
1086     }
1087 
1088     if (node_name) {
1089         rc = clear_rsc_failures(out, controld_api, node_name, NULL,
1090                                 operation, interval_spec, scheduler);
1091         if (rc != pcmk_rc_ok) {
1092             out->err(out, "Cleaned all resource failures on %s, but unable to clean history: %s",
1093                      node_name, pcmk_rc_str(rc));
1094             return rc;
1095         }
1096     } else {
1097         for (GList *iter = scheduler->nodes; iter; iter = iter->next) {
1098             pcmk_node_t *node = (pcmk_node_t *) iter->data;
1099 
1100             rc = clear_rsc_failures(out, controld_api, node->priv->name,
1101                                     NULL, operation, interval_spec, scheduler);
1102             if (rc != pcmk_rc_ok) {
1103                 out->err(out, "Cleaned all resource failures on all nodes, but unable to clean history: %s",
1104                          pcmk_rc_str(rc));
1105                 return rc;
1106             }
1107         }
1108     }
1109 
1110     out->info(out, "Cleaned up all resources on %s", display_name);
1111     return rc;
1112 }
1113 
1114 static void
1115 check_role(resource_checks_t *checks)
     /*  */
1116 {
1117     const char *role_s = g_hash_table_lookup(checks->rsc->priv->meta,
1118                                              PCMK_META_TARGET_ROLE);
1119 
1120     if (role_s == NULL) {
1121         return;
1122     }
1123     switch (pcmk_parse_role(role_s)) {
1124         case pcmk_role_stopped:
1125             checks->flags |= rsc_remain_stopped;
1126             break;
1127 
1128         case pcmk_role_unpromoted:
1129             if (pcmk_is_set(pe__const_top_resource(checks->rsc, false)->flags,
1130                             pcmk__rsc_promotable)) {
1131                 checks->flags |= rsc_unpromotable;
1132             }
1133             break;
1134 
1135         default:
1136             break;
1137     }
1138 }
1139 
1140 static void
1141 check_managed(resource_checks_t *checks)
     /*  */
1142 {
1143     const char *managed_s = g_hash_table_lookup(checks->rsc->priv->meta,
1144                                                 PCMK_META_IS_MANAGED);
1145 
1146     if ((managed_s != NULL) && !crm_is_true(managed_s)) {
1147         checks->flags |= rsc_unmanaged;
1148     }
1149 }
1150 
1151 static void
1152 check_locked(resource_checks_t *checks)
     /*  */
1153 {
1154     const pcmk_node_t *lock_node = checks->rsc->priv->lock_node;
1155 
1156     if (lock_node != NULL) {
1157         checks->flags |= rsc_locked;
1158         checks->lock_node = lock_node->priv->name;
1159     }
1160 }
1161 
1162 static bool
1163 node_is_unhealthy(pcmk_node_t *node)
     /*  */
1164 {
1165     switch (pe__health_strategy(node->priv->scheduler)) {
1166         case pcmk__health_strategy_none:
1167             break;
1168 
1169         case pcmk__health_strategy_no_red:
1170             if (pe__node_health(node) < 0) {
1171                 return true;
1172             }
1173             break;
1174 
1175         case pcmk__health_strategy_only_green:
1176             if (pe__node_health(node) <= 0) {
1177                 return true;
1178             }
1179             break;
1180 
1181         case pcmk__health_strategy_progressive:
1182         case pcmk__health_strategy_custom:
1183             /* @TODO These are finite scores, possibly with rules, and possibly
1184              * combining with other scores, so attributing these as a cause is
1185              * nontrivial.
1186              */
1187             break;
1188     }
1189     return false;
1190 }
1191 
1192 static void
1193 check_node_health(resource_checks_t *checks, pcmk_node_t *node)
     /*  */
1194 {
1195     if (node == NULL) {
1196         GHashTableIter iter;
1197         bool allowed = false;
1198         bool all_nodes_unhealthy = true;
1199 
1200         g_hash_table_iter_init(&iter, checks->rsc->priv->allowed_nodes);
1201         while (g_hash_table_iter_next(&iter, NULL, (void **) &node)) {
1202             allowed = true;
1203             if (!node_is_unhealthy(node)) {
1204                 all_nodes_unhealthy = false;
1205                 break;
1206             }
1207         }
1208         if (allowed && all_nodes_unhealthy) {
1209             checks->flags |= rsc_node_health;
1210         }
1211 
1212     } else if (node_is_unhealthy(node)) {
1213         checks->flags |= rsc_node_health;
1214     }
1215 }
1216 
1217 int
1218 cli_resource_check(pcmk__output_t *out, pcmk_resource_t *rsc, pcmk_node_t *node)
     /*  */
1219 {
1220     resource_checks_t checks = { .rsc = rsc };
1221 
1222     check_role(&checks);
1223     check_managed(&checks);
1224     check_locked(&checks);
1225     check_node_health(&checks, node);
1226 
1227     return out->message(out, "resource-check-list", &checks);
1228 }
1229 
1230 // \return Standard Pacemaker return code
1231 int
1232 cli_resource_fail(pcmk_ipc_api_t *controld_api, const char *host_uname,
     /*  */
1233                   const char *rsc_id, pcmk_scheduler_t *scheduler)
1234 {
1235     crm_notice("Failing %s on %s", rsc_id, host_uname);
1236     return send_lrm_rsc_op(controld_api, true, host_uname, rsc_id, scheduler);
1237 }
1238 
1239 static GHashTable *
1240 generate_resource_params(pcmk_resource_t *rsc, pcmk_node_t *node,
     /*  */
1241                          pcmk_scheduler_t *scheduler)
1242 {
1243     GHashTable *params = NULL;
1244     GHashTable *meta = NULL;
1245     GHashTable *combined = NULL;
1246     GHashTableIter iter;
1247     char *key = NULL;
1248     char *value = NULL;
1249 
1250     combined = pcmk__strkey_table(free, free);
1251 
1252     params = pe_rsc_params(rsc, node, scheduler);
1253     if (params != NULL) {
1254         g_hash_table_iter_init(&iter, params);
1255         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1256             pcmk__insert_dup(combined, key, value);
1257         }
1258     }
1259 
1260     meta = pcmk__strkey_table(free, free);
1261     get_meta_attributes(meta, rsc, NULL, scheduler);
1262     if (meta != NULL) {
1263         g_hash_table_iter_init(&iter, meta);
1264         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1265             char *crm_name = crm_meta_name(key);
1266 
1267             g_hash_table_insert(combined, crm_name, strdup(value));
1268         }
1269         g_hash_table_destroy(meta);
1270     }
1271 
1272     return combined;
1273 }
1274 
1275 bool resource_is_running_on(pcmk_resource_t *rsc, const char *host)
     /*  */
1276 {
1277     bool found = true;
1278     GList *hIter = NULL;
1279     GList *hosts = NULL;
1280 
1281     if (rsc == NULL) {
1282         return false;
1283     }
1284 
1285     rsc->priv->fns->location(rsc, &hosts, pcmk__rsc_node_current);
1286     for (hIter = hosts; host != NULL && hIter != NULL; hIter = hIter->next) {
1287         pcmk_node_t *node = (pcmk_node_t *) hIter->data;
1288 
1289         if (pcmk__strcase_any_of(host, node->priv->name, node->priv->id,
1290                                  NULL)) {
1291             crm_trace("Resource %s is running on %s\n", rsc->id, host);
1292             goto done;
1293         }
1294     }
1295 
1296     if (host != NULL) {
1297         crm_trace("Resource %s is not running on: %s\n", rsc->id, host);
1298         found = false;
1299 
1300     } else if(host == NULL && hosts == NULL) {
1301         crm_trace("Resource %s is not running\n", rsc->id);
1302         found = false;
1303     }
1304 
1305   done:
1306     g_list_free(hosts);
1307     return found;
1308 }
1309 
1310 /*!
1311  * \internal
1312  * \brief Create a list of all resources active on host from a given list
1313  *
1314  * \param[in] host      Name of host to check whether resources are active
1315  * \param[in] rsc_list  List of resources to check
1316  *
1317  * \return New list of resources from list that are active on host
1318  */
1319 static GList *
1320 get_active_resources(const char *host, GList *rsc_list)
     /*  */
1321 {
1322     GList *rIter = NULL;
1323     GList *active = NULL;
1324 
1325     for (rIter = rsc_list; rIter != NULL; rIter = rIter->next) {
1326         pcmk_resource_t *rsc = (pcmk_resource_t *) rIter->data;
1327 
1328         /* Expand groups to their members, because if we're restarting a member
1329          * other than the first, we can't otherwise tell which resources are
1330          * stopping and starting.
1331          */
1332         if (pcmk__is_group(rsc)) {
1333             GList *member_active = NULL;
1334 
1335             member_active = get_active_resources(host, rsc->priv->children);
1336             active = g_list_concat(active, member_active);
1337         } else if (resource_is_running_on(rsc, host)) {
1338             active = g_list_append(active, strdup(rsc->id));
1339         }
1340     }
1341     return active;
1342 }
1343 
1344 static void dump_list(GList *items, const char *tag)
     /*  */
1345 {
1346     int lpc = 0;
1347     GList *item = NULL;
1348 
1349     for (item = items; item != NULL; item = item->next) {
1350         crm_trace("%s[%d]: %s", tag, lpc, (char*)item->data);
1351         lpc++;
1352     }
1353 }
1354 
1355 static void display_list(pcmk__output_t *out, GList *items, const char *tag)
     /*  */
1356 {
1357     GList *item = NULL;
1358 
1359     for (item = items; item != NULL; item = item->next) {
1360         out->info(out, "%s%s", tag, (const char *)item->data);
1361     }
1362 }
1363 
1364 /*!
1365  * \internal
1366  * \brief Update scheduler XML input based on a CIB query and the current time
1367  *
1368  * The CIB XML is upgraded to the latest schema version.
1369  *
1370  * \param[in,out] out           Output object
1371  * \param[in,out] scheduler     Scheduler data to update
1372  * \param[in]     cib           Connection to the CIB manager
1373  * \param[out]    cib_xml_orig  Where to store CIB XML before any schema
1374  *                              upgrades (can be \c NULL)
1375  *
1376  * \return Standard Pacemaker return code
1377  */
1378 int
1379 update_scheduler_input(pcmk__output_t *out, pcmk_scheduler_t *scheduler,
     /*  */
1380                        cib_t *cib, xmlNode **cib_xml_orig)
1381 {
1382     xmlNode *queried_xml = NULL;
1383     xmlNode *updated_xml = NULL;
1384     int rc = pcmk_rc_ok;
1385 
1386     pcmk__assert((out != NULL) && (scheduler != NULL)
1387                  && (scheduler->input == NULL) && (scheduler->priv->now == NULL)
1388                  && (cib != NULL)
1389                  && ((cib_xml_orig == NULL) || (*cib_xml_orig == NULL)));
1390 
1391     rc = cib->cmds->query(cib, NULL, &queried_xml, cib_sync_call);
1392     rc = pcmk_legacy2rc(rc);
1393     if (rc != pcmk_rc_ok) {
1394         out->err(out, "Could not obtain the current CIB: %s", pcmk_rc_str(rc));
1395         goto done;
1396     }
1397 
1398     if (cib_xml_orig != NULL) {
1399         updated_xml = pcmk__xml_copy(NULL, queried_xml);
1400     } else {
1401         // No need to preserve the pre-upgrade CIB, so don't make a copy
1402         updated_xml = queried_xml;
1403         queried_xml = NULL;
1404     }
1405 
1406     rc = pcmk__update_configured_schema(&updated_xml, false);
1407     if (rc != pcmk_rc_ok) {
1408         out->err(out, "Could not upgrade the current CIB XML: %s",
1409                  pcmk_rc_str(rc));
1410         pcmk__xml_free(updated_xml);
1411         goto done;
1412     }
1413 
1414     scheduler->input = updated_xml;
1415     scheduler->priv->now = crm_time_new(NULL);
1416 
1417 done:
1418     if ((rc == pcmk_rc_ok) && (cib_xml_orig != NULL)) {
1419         *cib_xml_orig = queried_xml;
1420     } else {
1421         pcmk__xml_free(queried_xml);
1422     }
1423     return rc;
1424 }
1425 
1426 // \return Standard Pacemaker return code
1427 static int
1428 update_dataset(cib_t *cib, pcmk_scheduler_t *scheduler, xmlNode **cib_xml_orig,
     /*  */
1429                bool simulate)
1430 {
1431     char *pid = NULL;
1432     char *shadow_file = NULL;
1433     cib_t *shadow_cib = NULL;
1434     int rc = pcmk_rc_ok;
1435 
1436     pcmk__output_t *out = scheduler->priv->out;
1437 
1438     pe_reset_working_set(scheduler);
1439     pcmk__set_scheduler_flags(scheduler, pcmk__sched_no_counts);
1440 
1441     if(simulate) {
1442         bool prev_quiet = false;
1443 
1444         rc = update_scheduler_input(out, scheduler, cib, NULL);
1445         if (rc != pcmk_rc_ok) {
1446             goto done;
1447         }
1448 
1449         pid = pcmk__getpid_s();
1450         shadow_cib = cib_shadow_new(pid);
1451         shadow_file = get_shadow_file(pid);
1452 
1453         if (shadow_cib == NULL) {
1454             out->err(out, "Could not create shadow cib: '%s'", pid);
1455             rc = ENXIO;
1456             goto done;
1457         }
1458 
1459         rc = pcmk__xml_write_file(scheduler->input, shadow_file, false);
1460         if (rc != pcmk_rc_ok) {
1461             out->err(out, "Could not populate shadow cib: %s", pcmk_rc_str(rc));
1462             goto done;
1463         }
1464 
1465         rc = shadow_cib->cmds->signon(shadow_cib, crm_system_name, cib_command);
1466         rc = pcmk_legacy2rc(rc);
1467 
1468         if (rc != pcmk_rc_ok) {
1469             out->err(out, "Could not connect to shadow cib: %s",
1470                      pcmk_rc_str(rc));
1471             goto done;
1472         }
1473 
1474         pcmk__schedule_actions(scheduler->input, pcmk__sched_no_counts,
1475                                scheduler);
1476 
1477         prev_quiet = out->is_quiet(out);
1478         out->quiet = true;
1479         pcmk__simulate_transition(scheduler, shadow_cib, NULL);
1480         out->quiet = prev_quiet;
1481 
1482         rc = update_dataset(shadow_cib, scheduler, cib_xml_orig, false);
1483 
1484     } else {
1485         xmlNode *xml = NULL;
1486 
1487         rc = update_scheduler_input(out, scheduler, cib, &xml);
1488         if (rc != pcmk_rc_ok) {
1489             goto done;
1490         }
1491 
1492         pcmk__xml_free(*cib_xml_orig);
1493         *cib_xml_orig = xml;
1494         cluster_status(scheduler);
1495     }
1496 
1497   done:
1498     // Do not free scheduler->input because rsc->priv->xml must remain valid
1499     cib_delete(shadow_cib);
1500     free(pid);
1501 
1502     if(shadow_file) {
1503         unlink(shadow_file);
1504         free(shadow_file);
1505     }
1506 
1507     return rc;
1508 }
1509 
1510 /*!
1511  * \internal
1512  * \brief Find the maximum stop timeout of a resource and its children (if any)
1513  *
1514  * \param[in,out] rsc  Resource to get timeout for
1515  *
1516  * \return Maximum stop timeout for \p rsc (in milliseconds)
1517  */
1518 static guint
1519 max_rsc_stop_timeout(pcmk_resource_t *rsc)
     /*  */
1520 {
1521     long long result_ll;
1522     guint max_delay = 0;
1523     xmlNode *config = NULL;
1524     GHashTable *meta = NULL;
1525 
1526     if (rsc == NULL) {
1527         return 0;
1528     }
1529 
1530     // If resource is collective, use maximum of its children's stop timeouts
1531     if (rsc->priv->children != NULL) {
1532 
1533         for (GList *iter = rsc->priv->children;
1534              iter != NULL; iter = iter->next) {
1535 
1536             pcmk_resource_t *child = iter->data;
1537             guint delay = max_rsc_stop_timeout(child);
1538 
1539             if (delay > max_delay) {
1540                 pcmk__rsc_trace(rsc,
1541                                 "Maximum stop timeout for %s is now %s "
1542                                 "due to %s", rsc->id,
1543                                 pcmk__readable_interval(delay), child->id);
1544                 max_delay = delay;
1545             }
1546         }
1547         return max_delay;
1548     }
1549 
1550     // Get resource's stop action configuration from CIB
1551     config = pcmk__find_action_config(rsc, PCMK_ACTION_STOP, 0, true);
1552 
1553     /* Get configured timeout for stop action (fully evaluated for rules,
1554      * defaults, etc.).
1555      *
1556      * @TODO This currently ignores node (which might matter for rules)
1557      */
1558     meta = pcmk__unpack_action_meta(rsc, NULL, PCMK_ACTION_STOP, 0, config);
1559     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
1560                        &result_ll, -1LL) == pcmk_rc_ok) && (result_ll >= 0)) {
1561         max_delay = (guint) QB_MIN(result_ll, UINT_MAX);
1562     }
1563     g_hash_table_destroy(meta);
1564 
1565     return max_delay;
1566 }
1567 
1568 /*!
1569  * \internal
1570  * \brief Find a reasonable waiting time for stopping any one resource in a list
1571  *
1572  * \param[in,out] scheduler  Scheduler data
1573  * \param[in]     resources  List of names of resources that will be stopped
1574  *
1575  * \return Rough estimate of a reasonable time to wait (in seconds) to stop any
1576  *         one resource in \p resources
1577  * \note This estimate is very rough, simply the maximum stop timeout of all
1578  *       given resources and their children, plus a small fudge factor. It does
1579  *       not account for children that must be stopped in sequence, action
1580  *       throttling, or any demotions needed. It checks the stop timeout, even
1581  *       if the resources in question are actually being started.
1582  */
1583 static guint
1584 wait_time_estimate(pcmk_scheduler_t *scheduler, const GList *resources)
     /*  */
1585 {
1586     guint max_delay = 0U;
1587 
1588     // Find maximum stop timeout in milliseconds
1589     for (const GList *item = resources; item != NULL; item = item->next) {
1590         pcmk_resource_t *rsc = pe_find_resource(scheduler->priv->resources,
1591                                                 (const char *) item->data);
1592         guint delay = max_rsc_stop_timeout(rsc);
1593 
1594         if (delay > max_delay) {
1595             pcmk__rsc_trace(rsc,
1596                             "Wait time is now %s due to %s",
1597                             pcmk__readable_interval(delay), rsc->id);
1598             max_delay = delay;
1599         }
1600     }
1601 
1602     return pcmk__timeout_ms2s(max_delay) + 5;
1603 }
1604 
1605 #define waiting_for_starts(d, r, h) ((d != NULL) || \
1606                                     (!resource_is_running_on((r), (h))))
1607 
1608 /*!
1609  * \internal
1610  * \brief Restart a resource (on a particular host if requested).
1611  *
1612  * \param[in,out] out                 Output object
1613  * \param[in,out] rsc                 The resource to restart
1614  * \param[in]     node                Node to restart resource on (NULL for all)
1615  * \param[in]     move_lifetime       If not NULL, how long constraint should
1616  *                                    remain in effect (as ISO 8601 string)
1617  * \param[in]     timeout_ms          Consider failed if actions do not complete
1618  *                                    in this time (specified in milliseconds,
1619  *                                    but a two-second granularity is actually
1620  *                                    used; if 0, it will be calculated based on
1621  *                                    the resource timeout)
1622  * \param[in,out] cib                 Connection to the CIB manager
1623  * \param[in]     promoted_role_only  If true, limit to promoted instances
1624  * \param[in]     force               If true, apply only to requested instance
1625  *                                    if part of a collective resource
1626  *
1627  * \return Standard Pacemaker return code (exits on certain failures)
1628  */
1629 int
1630 cli_resource_restart(pcmk__output_t *out, pcmk_resource_t *rsc,
     /*  */
1631                      const pcmk_node_t *node, const char *move_lifetime,
1632                      guint timeout_ms, cib_t *cib, gboolean promoted_role_only,
1633                      gboolean force)
1634 {
1635     int rc = pcmk_rc_ok;
1636     int lpc = 0;
1637     int before = 0;
1638     guint step_timeout_s = 0;
1639     guint sleep_interval = 2U;
1640     guint timeout = pcmk__timeout_ms2s(timeout_ms);
1641 
1642     bool stop_via_ban = false;
1643     char *rsc_id = NULL;
1644     char *lookup_id = NULL;
1645     char *orig_target_role = NULL;
1646     xmlNode *cib_xml_orig = NULL;
1647 
1648     GList *list_delta = NULL;
1649     GList *target_active = NULL;
1650     GList *current_active = NULL;
1651     GList *restart_target_active = NULL;
1652 
1653     pcmk_scheduler_t *scheduler = NULL;
1654     pcmk_resource_t *parent = uber_parent(rsc);
1655 
1656     bool running = false;
1657     const char *id = pcmk__s(rsc->priv->history_id, rsc->id);
1658     const char *host = node ? node->priv->name : NULL;
1659 
1660     /* If the implicit resource or primitive resource of a bundle is given, operate on the
1661      * bundle itself instead.
1662      */
1663     if (pcmk__is_bundled(rsc)) {
1664         rsc = parent->priv->parent;
1665     }
1666 
1667     running = resource_is_running_on(rsc, host);
1668 
1669     if (pcmk__is_clone(parent) && !running) {
1670         if (pcmk__is_unique_clone(parent)) {
1671             lookup_id = strdup(rsc->id);
1672         } else {
1673             lookup_id = clone_strip(rsc->id);
1674         }
1675 
1676         rsc = parent->priv->fns->find_rsc(parent, lookup_id, node,
1677                                           pcmk_rsc_match_basename
1678                                           |pcmk_rsc_match_current_node);
1679         free(lookup_id);
1680         running = resource_is_running_on(rsc, host);
1681     }
1682 
1683     if (!running) {
1684         if (host) {
1685             out->err(out, "%s is not running on %s and so cannot be restarted", id, host);
1686         } else {
1687             out->err(out, "%s is not running anywhere and so cannot be restarted", id);
1688         }
1689         return ENXIO;
1690     }
1691 
1692     if (!pcmk_is_set(rsc->flags, pcmk__rsc_managed)) {
1693         out->err(out, "Unmanaged resources cannot be restarted.");
1694         return EAGAIN;
1695     }
1696 
1697     rsc_id = strdup(rsc->id);
1698 
1699     if (pcmk__is_unique_clone(parent)) {
1700         lookup_id = strdup(rsc->id);
1701     } else {
1702         lookup_id = clone_strip(rsc->id);
1703     }
1704 
1705     if (host) {
1706         if (pcmk__is_clone(rsc) || pe_bundle_replicas(rsc)) {
1707             stop_via_ban = true;
1708         } else if (pcmk__is_clone(parent)) {
1709             stop_via_ban = true;
1710             free(lookup_id);
1711             lookup_id = strdup(parent->id);
1712         }
1713     }
1714 
1715     /*
1716       grab full cib
1717       determine originally active resources
1718       disable or ban
1719       poll cib and watch for affected resources to get stopped
1720       without --timeout, calculate the stop timeout for each step and wait for that
1721       if we hit --timeout or the service timeout, re-enable or un-ban, report failure and indicate which resources we couldn't take down
1722       if everything stopped, re-enable or un-ban
1723       poll cib and watch for affected resources to get started
1724       without --timeout, calculate the start timeout for each step and wait for that
1725       if we hit --timeout or the service timeout, report (different) failure and indicate which resources we couldn't bring back up
1726       report success
1727 
1728       Optimizations:
1729       - use constraints to determine ordered list of affected resources
1730       - Allow a --no-deps option (aka. --force-restart)
1731     */
1732 
1733     scheduler = pe_new_working_set();
1734     if (scheduler == NULL) {
1735         rc = errno;
1736         out->err(out, "Could not allocate scheduler data: %s", pcmk_rc_str(rc));
1737         goto done;
1738     }
1739 
1740     scheduler->priv->out = out;
1741     rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
1742 
1743     if(rc != pcmk_rc_ok) {
1744         out->err(out, "Could not get new resource list: %s (%d)", pcmk_rc_str(rc), rc);
1745         goto done;
1746     }
1747 
1748     restart_target_active = get_active_resources(host,
1749                                                  scheduler->priv->resources);
1750     current_active = get_active_resources(host, scheduler->priv->resources);
1751 
1752     dump_list(current_active, "Origin");
1753 
1754     if (stop_via_ban) {
1755         /* Stop the clone or bundle instance by banning it from the host */
1756         out->quiet = true;
1757         rc = cli_resource_ban(out, lookup_id, host, move_lifetime, cib,
1758                               promoted_role_only, PCMK_ROLE_PROMOTED);
1759     } else {
1760         xmlNode *xml_search = NULL;
1761 
1762         /* Stop the resource by setting PCMK_META_TARGET_ROLE to Stopped.
1763          * Remember any existing PCMK_META_TARGET_ROLE so we can restore it
1764          * later (though it only makes any difference if it's Unpromoted).
1765          */
1766 
1767         rc = find_resource_attr(out, cib, PCMK_XA_VALUE, lookup_id, NULL, NULL, NULL,
1768                                 PCMK_META_TARGET_ROLE, &xml_search);
1769 
1770         if (rc == pcmk_rc_ok) {
1771             orig_target_role = crm_element_value_copy(xml_search, PCMK_XA_VALUE);
1772         }
1773 
1774         pcmk__xml_free(xml_search);
1775 
1776         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
1777                                            PCMK_XE_META_ATTRIBUTES, NULL,
1778                                            PCMK_META_TARGET_ROLE,
1779                                            PCMK_ACTION_STOPPED, FALSE, cib,
1780                                            cib_xml_orig, force);
1781     }
1782     if(rc != pcmk_rc_ok) {
1783         out->err(out, "Could not set " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
1784                  rsc_id, pcmk_rc_str(rc), rc);
1785         if (current_active != NULL) {
1786             g_list_free_full(current_active, free);
1787             current_active = NULL;
1788         }
1789         if (restart_target_active != NULL) {
1790             g_list_free_full(restart_target_active, free);
1791             restart_target_active = NULL;
1792         }
1793         goto done;
1794     }
1795 
1796     rc = update_dataset(cib, scheduler, &cib_xml_orig, true);
1797     if(rc != pcmk_rc_ok) {
1798         out->err(out, "Could not determine which resources would be stopped");
1799         goto failure;
1800     }
1801 
1802     target_active = get_active_resources(host, scheduler->priv->resources);
1803     dump_list(target_active, "Target");
1804 
1805     list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1806     out->info(out, "Waiting for %d resources to stop:", g_list_length(list_delta));
1807     display_list(out, list_delta, " * ");
1808 
1809     step_timeout_s = timeout / sleep_interval;
1810     while (list_delta != NULL) {
1811         before = g_list_length(list_delta);
1812         if(timeout_ms == 0) {
1813             step_timeout_s = wait_time_estimate(scheduler, list_delta)
1814                              / sleep_interval;
1815         }
1816 
1817         /* We probably don't need the entire step timeout */
1818         for(lpc = 0; (lpc < step_timeout_s) && (list_delta != NULL); lpc++) {
1819             sleep(sleep_interval);
1820             if(timeout) {
1821                 timeout -= sleep_interval;
1822                 crm_trace("%us remaining", timeout);
1823             }
1824             rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
1825             if(rc != pcmk_rc_ok) {
1826                 out->err(out, "Could not determine which resources were stopped");
1827                 goto failure;
1828             }
1829 
1830             if (current_active != NULL) {
1831                 g_list_free_full(current_active, free);
1832             }
1833             current_active = get_active_resources(host,
1834                                                   scheduler->priv->resources);
1835 
1836             g_list_free(list_delta);
1837             list_delta = pcmk__subtract_lists(current_active, target_active, (GCompareFunc) strcmp);
1838 
1839             dump_list(current_active, "Current");
1840             dump_list(list_delta, "Delta");
1841         }
1842 
1843         crm_trace("%d (was %d) resources remaining", g_list_length(list_delta), before);
1844         if(before == g_list_length(list_delta)) {
1845             /* aborted during stop phase, print the contents of list_delta */
1846             out->err(out, "Could not complete shutdown of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1847             display_list(out, list_delta, " * ");
1848             rc = ETIME;
1849             goto failure;
1850         }
1851 
1852     }
1853 
1854     if (stop_via_ban) {
1855         rc = cli_resource_clear(lookup_id, host, NULL, cib, true, force);
1856 
1857     } else if (orig_target_role) {
1858         rc = cli_resource_update_attribute(rsc, rsc_id, NULL,
1859                                            PCMK_XE_META_ATTRIBUTES, NULL,
1860                                            PCMK_META_TARGET_ROLE,
1861                                            orig_target_role, FALSE, cib,
1862                                            cib_xml_orig, force);
1863         free(orig_target_role);
1864         orig_target_role = NULL;
1865     } else {
1866         rc = cli_resource_delete_attribute(rsc, rsc_id, NULL,
1867                                            PCMK_XE_META_ATTRIBUTES, NULL,
1868                                            PCMK_META_TARGET_ROLE, cib,
1869                                            cib_xml_orig, force);
1870     }
1871 
1872     if(rc != pcmk_rc_ok) {
1873         out->err(out,
1874                  "Could not unset " PCMK_META_TARGET_ROLE " for %s: %s (%d)",
1875                  rsc_id, pcmk_rc_str(rc), rc);
1876         goto done;
1877     }
1878 
1879     if (target_active != NULL) {
1880         g_list_free_full(target_active, free);
1881     }
1882     target_active = restart_target_active;
1883 
1884     list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1885     out->info(out, "Waiting for %d resources to start again:", g_list_length(list_delta));
1886     display_list(out, list_delta, " * ");
1887 
1888     step_timeout_s = timeout / sleep_interval;
1889     while (waiting_for_starts(list_delta, rsc, host)) {
1890         before = g_list_length(list_delta);
1891         if(timeout_ms == 0) {
1892             step_timeout_s = wait_time_estimate(scheduler, list_delta)
1893                              / sleep_interval;
1894         }
1895 
1896         /* We probably don't need the entire step timeout */
1897         for (lpc = 0; (lpc < step_timeout_s) && waiting_for_starts(list_delta, rsc, host); lpc++) {
1898 
1899             sleep(sleep_interval);
1900             if(timeout) {
1901                 timeout -= sleep_interval;
1902                 crm_trace("%ds remaining", timeout);
1903             }
1904 
1905             rc = update_dataset(cib, scheduler, &cib_xml_orig, false);
1906             if(rc != pcmk_rc_ok) {
1907                 out->err(out, "Could not determine which resources were started");
1908                 goto failure;
1909             }
1910 
1911             /* It's OK if dependent resources moved to a different node,
1912              * so we check active resources on all nodes.
1913              */
1914             if (current_active != NULL) {
1915                 g_list_free_full(current_active, free);
1916             }
1917             current_active = get_active_resources(NULL,
1918                                                   scheduler->priv->resources);
1919 
1920             g_list_free(list_delta);
1921             list_delta = pcmk__subtract_lists(target_active, current_active, (GCompareFunc) strcmp);
1922             dump_list(current_active, "Current");
1923             dump_list(list_delta, "Delta");
1924         }
1925 
1926         if(before == g_list_length(list_delta)) {
1927             /* aborted during start phase, print the contents of list_delta */
1928             out->err(out, "Could not complete restart of %s, %d resources remaining", rsc_id, g_list_length(list_delta));
1929             display_list(out, list_delta, " * ");
1930             rc = ETIME;
1931             goto failure;
1932         }
1933 
1934     }
1935 
1936     rc = pcmk_rc_ok;
1937     goto done;
1938 
1939   failure:
1940     if (stop_via_ban) {
1941         cli_resource_clear(lookup_id, host, NULL, cib, true, force);
1942     } else if (orig_target_role) {
1943         cli_resource_update_attribute(rsc, rsc_id, NULL,
1944                                       PCMK_XE_META_ATTRIBUTES, NULL,
1945                                       PCMK_META_TARGET_ROLE, orig_target_role,
1946                                       FALSE, cib, cib_xml_orig, force);
1947         free(orig_target_role);
1948     } else {
1949         cli_resource_delete_attribute(rsc, rsc_id, NULL,
1950                                       PCMK_XE_META_ATTRIBUTES, NULL,
1951                                       PCMK_META_TARGET_ROLE, cib, cib_xml_orig,
1952                                       force);
1953     }
1954 
1955 done:
1956     if (list_delta != NULL) {
1957         g_list_free(list_delta);
1958     }
1959     if (current_active != NULL) {
1960         g_list_free_full(current_active, free);
1961     }
1962     if (target_active != NULL && (target_active != restart_target_active)) {
1963         g_list_free_full(target_active, free);
1964     }
1965     if (restart_target_active != NULL) {
1966         g_list_free_full(restart_target_active, free);
1967     }
1968     free(rsc_id);
1969     free(lookup_id);
1970     pe_free_working_set(scheduler);
1971     return rc;
1972 }
1973 
1974 static inline bool
1975 action_is_pending(const pcmk_action_t *action)
     /*  */
1976 {
1977     if (pcmk_any_flags_set(action->flags,
1978                            pcmk__action_optional|pcmk__action_pseudo)
1979         || !pcmk_is_set(action->flags, pcmk__action_runnable)
1980         || pcmk__str_eq(PCMK_ACTION_NOTIFY, action->task, pcmk__str_casei)) {
1981         return false;
1982     }
1983     return true;
1984 }
1985 
1986 /*!
1987  * \internal
1988  * \brief Check whether any actions in a list are pending
1989  *
1990  * \param[in] actions   List of actions to check
1991  *
1992  * \return true if any actions in the list are pending, otherwise false
1993  */
1994 static bool
1995 actions_are_pending(const GList *actions)
     /*  */
1996 {
1997     for (const GList *action = actions; action != NULL; action = action->next) {
1998         const pcmk_action_t *a = (const pcmk_action_t *) action->data;
1999 
2000         if (action_is_pending(a)) {
2001             crm_notice("Waiting for %s (flags=%#.8x)", a->uuid, a->flags);
2002             return true;
2003         }
2004     }
2005     return false;
2006 }
2007 
2008 static void
2009 print_pending_actions(pcmk__output_t *out, GList *actions)
     /*  */
2010 {
2011     GList *action;
2012 
2013     out->info(out, "Pending actions:");
2014     for (action = actions; action != NULL; action = action->next) {
2015         pcmk_action_t *a = (pcmk_action_t *) action->data;
2016 
2017         if (!action_is_pending(a)) {
2018             continue;
2019         }
2020 
2021         if (a->node) {
2022             out->info(out, "\tAction %d: %s\ton %s",
2023                       a->id, a->uuid, pcmk__node_name(a->node));
2024         } else {
2025             out->info(out, "\tAction %d: %s", a->id, a->uuid);
2026         }
2027     }
2028 }
2029 
2030 /* For --wait, timeout (in seconds) to use if caller doesn't specify one */
2031 #define WAIT_DEFAULT_TIMEOUT_S (60 * 60)
2032 
2033 /* For --wait, how long to sleep between cluster state checks */
2034 #define WAIT_SLEEP_S (2)
2035 
2036 /*!
2037  * \internal
2038  * \brief Wait until all pending cluster actions are complete
2039  *
2040  * This waits until either the CIB's transition graph is idle or a timeout is
2041  * reached.
2042  *
2043  * \param[in,out] out          Output object
2044  * \param[in]     timeout_ms   Consider failed if actions do not complete in
2045  *                             this time (specified in milliseconds, but
2046  *                             one-second granularity is actually used; if 0, a
2047  *                             default will be used)
2048  * \param[in,out] cib          Connection to the CIB manager
2049  *
2050  * \return Standard Pacemaker return code
2051  */
2052 int
2053 wait_till_stable(pcmk__output_t *out, guint timeout_ms, cib_t * cib)
     /*  */
2054 {
2055     pcmk_scheduler_t *scheduler = NULL;
2056     xmlXPathObjectPtr search;
2057     int rc = pcmk_rc_ok;
2058     bool pending_unknown_state_resources;
2059     time_t expire_time = time(NULL);
2060     time_t time_diff;
2061     bool printed_version_warning = out->is_quiet(out); // i.e. don't print if quiet
2062     char *xpath = NULL;
2063 
2064     if (timeout_ms == 0) {
2065         expire_time += WAIT_DEFAULT_TIMEOUT_S;
2066     } else {
2067         expire_time += pcmk__timeout_ms2s(timeout_ms + 999);
2068     }
2069 
2070     scheduler = pe_new_working_set();
2071     if (scheduler == NULL) {
2072         return ENOMEM;
2073     }
2074 
2075     xpath = crm_strdup_printf("/" PCMK_XE_CIB "/" PCMK_XE_STATUS
2076                               "/" PCMK__XE_NODE_STATE "/" PCMK__XE_LRM
2077                               "/" PCMK__XE_LRM_RESOURCES
2078                               "/" PCMK__XE_LRM_RESOURCE
2079                               "/" PCMK__XE_LRM_RSC_OP
2080                               "[@" PCMK__XA_RC_CODE "='%d']",
2081                               PCMK_OCF_UNKNOWN);
2082     do {
2083         /* Abort if timeout is reached */
2084         time_diff = expire_time - time(NULL);
2085         if (time_diff <= 0) {
2086             print_pending_actions(out, scheduler->priv->actions);
2087             rc = ETIME;
2088             break;
2089         }
2090 
2091         crm_info("Waiting up to %lld seconds for cluster actions to complete",
2092                  (long long) time_diff);
2093 
2094         if (rc == pcmk_rc_ok) { /* this avoids sleep on first loop iteration */
2095             sleep(WAIT_SLEEP_S);
2096         }
2097 
2098         /* Get latest transition graph */
2099         pe_reset_working_set(scheduler);
2100         rc = update_scheduler_input(out, scheduler, cib, NULL);
2101         if (rc != pcmk_rc_ok) {
2102             break;
2103         }
2104         pcmk__schedule_actions(scheduler->input, pcmk__sched_no_counts,
2105                                scheduler);
2106 
2107         if (!printed_version_warning) {
2108             /* If the DC has a different version than the local node, the two
2109              * could come to different conclusions about what actions need to be
2110              * done. Warn the user in this case.
2111              *
2112              * @TODO A possible long-term solution would be to reimplement the
2113              * wait as a new controller operation that would be forwarded to the
2114              * DC. However, that would have potential problems of its own.
2115              */
2116             const char *dc_version = NULL;
2117 
2118             dc_version = g_hash_table_lookup(scheduler->priv->options,
2119                                              PCMK_OPT_DC_VERSION);
2120             if (!pcmk__str_eq(dc_version, PACEMAKER_VERSION "-" BUILD_VERSION, pcmk__str_casei)) {
2121                 out->info(out, "warning: wait option may not work properly in "
2122                           "mixed-version cluster");
2123                 printed_version_warning = true;
2124             }
2125         }
2126 
2127         search = xpath_search(scheduler->input, xpath);
2128         pending_unknown_state_resources = (numXpathResults(search) > 0);
2129         freeXpathObject(search);
2130     } while (actions_are_pending(scheduler->priv->actions)
2131              || pending_unknown_state_resources);
2132 
2133     pe_free_working_set(scheduler);
2134     free(xpath);
2135     return rc;
2136 }
2137 
2138 static const char *
2139 get_action(const char *rsc_action) {
     /*  */
2140     const char *action = NULL;
2141 
2142     if (pcmk__str_eq(rsc_action, "validate", pcmk__str_casei)) {
2143         action = PCMK_ACTION_VALIDATE_ALL;
2144 
2145     } else if (pcmk__str_eq(rsc_action, "force-check", pcmk__str_casei)) {
2146         action = PCMK_ACTION_MONITOR;
2147 
2148     } else if (pcmk__strcase_any_of(rsc_action, "force-start", "force-stop",
2149                                     "force-demote", "force-promote", NULL)) {
2150         action = rsc_action+6;
2151     } else {
2152         action = rsc_action;
2153     }
2154 
2155     return action;
2156 }
2157 
2158 /*!
2159  * \brief Set up environment variables as expected by resource agents
2160  *
2161  * When the cluster executes resource agents, it adds certain environment
2162  * variables (directly or via resource meta-attributes) expected by some
2163  * resource agents. Add the essential ones that many resource agents expect, so
2164  * the behavior is the same for command-line execution.
2165  *
2166  * \param[in,out] params       Resource parameters that will be passed to agent
2167  * \param[in]     timeout_ms   Action timeout (in milliseconds)
2168  * \param[in]     check_level  OCF check level
2169  * \param[in]     verbosity    Verbosity level
2170  */
2171 static void
2172 set_agent_environment(GHashTable *params, guint timeout_ms, int check_level,
     /*  */
2173                       int verbosity)
2174 {
2175     g_hash_table_insert(params, crm_meta_name(PCMK_META_TIMEOUT),
2176                         crm_strdup_printf("%u", timeout_ms));
2177 
2178     pcmk__insert_dup(params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
2179 
2180     if (check_level >= 0) {
2181         char *level = crm_strdup_printf("%d", check_level);
2182 
2183         setenv("OCF_CHECK_LEVEL", level, 1);
2184         free(level);
2185     }
2186 
2187     pcmk__set_env_option(PCMK__ENV_DEBUG, ((verbosity > 0)? "1" : "0"), true);
2188     if (verbosity > 1) {
2189         setenv("OCF_TRACE_RA", "1", 1);
2190     }
2191 
2192     /* A resource agent using the standard ocf-shellfuncs library will not print
2193      * messages to stderr if it doesn't have a controlling terminal (e.g. if
2194      * crm_resource is called via script or ssh). This forces it to do so.
2195      */
2196     setenv("OCF_TRACE_FILE", "/dev/stderr", 0);
2197 }
2198 
2199 /*!
2200  * \internal
2201  * \brief Apply command-line overrides to resource parameters
2202  *
2203  * \param[in,out] params     Parameters to be passed to agent
2204  * \param[in]     overrides  Parameters to override (or NULL if none)
2205  */
2206 static void
2207 apply_overrides(GHashTable *params, GHashTable *overrides)
     /*  */
2208 {
2209     if (overrides != NULL) {
2210         GHashTableIter iter;
2211         char *name = NULL;
2212         char *value = NULL;
2213 
2214         g_hash_table_iter_init(&iter, overrides);
2215         while (g_hash_table_iter_next(&iter, (gpointer *) &name,
2216                                       (gpointer *) &value)) {
2217             pcmk__insert_dup(params, name, value);
2218         }
2219     }
2220 }
2221 
2222 crm_exit_t
2223 cli_resource_execute_from_params(pcmk__output_t *out, const char *rsc_name,
     /*  */
2224                                  const char *rsc_class, const char *rsc_prov,
2225                                  const char *rsc_type, const char *rsc_action,
2226                                  GHashTable *params, GHashTable *override_hash,
2227                                  guint timeout_ms, int resource_verbose,
2228                                  gboolean force, int check_level)
2229 {
2230     const char *class = rsc_class;
2231     const char *action = get_action(rsc_action);
2232     crm_exit_t exit_code = CRM_EX_OK;
2233     svc_action_t *op = NULL;
2234 
2235     // If no timeout was provided, use the same default as the cluster
2236     if (timeout_ms == 0U) {
2237         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
2238     }
2239 
2240     set_agent_environment(params, timeout_ms, check_level, resource_verbose);
2241     apply_overrides(params, override_hash);
2242 
2243     op = services__create_resource_action(rsc_name? rsc_name : "test",
2244                                           rsc_class, rsc_prov, rsc_type, action,
2245                                           0, QB_MIN(timeout_ms, INT_MAX),
2246                                           params, 0);
2247     if (op == NULL) {
2248         out->err(out, "Could not execute %s using %s%s%s:%s: %s",
2249                  action, rsc_class, (rsc_prov? ":" : ""),
2250                  (rsc_prov? rsc_prov : ""), rsc_type, strerror(ENOMEM));
2251         g_hash_table_destroy(params);
2252         return CRM_EX_OSERR;
2253     }
2254 
2255 #if PCMK__ENABLE_SERVICE
2256     if (pcmk__str_eq(rsc_class, PCMK_RESOURCE_CLASS_SERVICE, pcmk__str_casei)) {
2257         class = resources_find_service_class(rsc_type);
2258     }
2259 #endif
2260 
2261     if (!pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_cli_exec)) {
2262         services__format_result(op, CRM_EX_UNIMPLEMENT_FEATURE, PCMK_EXEC_ERROR,
2263                                 "Manual execution of the %s standard is "
2264                                 "unsupported", pcmk__s(class, "unspecified"));
2265     }
2266 
2267     if (op->rc != PCMK_OCF_UNKNOWN) {
2268         exit_code = op->rc;
2269         goto done;
2270     }
2271 
2272     services_action_sync(op);
2273 
2274     // Map results to OCF codes for consistent reporting to user
2275     {
2276         enum ocf_exitcode ocf_code = services_result2ocf(class, action, op->rc);
2277 
2278         // Cast variable instead of function return to keep compilers happy
2279         exit_code = (crm_exit_t) ocf_code;
2280     }
2281 
2282 done:
2283     out->message(out, "resource-agent-action", resource_verbose, rsc_class,
2284                  rsc_prov, rsc_type, rsc_name, rsc_action, override_hash,
2285                  exit_code, op->status, services__exit_reason(op),
2286                  op->stdout_data, op->stderr_data);
2287     services_action_free(op);
2288     return exit_code;
2289 }
2290 
2291 /*!
2292  * \internal
2293  * \brief Get the timeout the cluster would use for an action
2294  *
2295  * \param[in] rsc     Resource that action is for
2296  * \param[in] action  Name of action
2297  */
2298 static guint
2299 get_action_timeout(pcmk_resource_t *rsc, const char *action)
     /*  */
2300 {
2301     long long timeout_ms = -1LL;
2302     xmlNode *op = pcmk__find_action_config(rsc, action, 0, true);
2303     GHashTable *meta = pcmk__unpack_action_meta(rsc, NULL, action, 0, op);
2304 
2305     if ((pcmk__scan_ll(g_hash_table_lookup(meta, PCMK_META_TIMEOUT),
2306                        &timeout_ms, -1LL) != pcmk_rc_ok)
2307         || (timeout_ms <= 0LL)) {
2308         timeout_ms = PCMK_DEFAULT_ACTION_TIMEOUT_MS;
2309     }
2310     g_hash_table_destroy(meta);
2311     return (guint) QB_MIN(timeout_ms, UINT_MAX);
2312 }
2313 
2314 crm_exit_t
2315 cli_resource_execute(pcmk_resource_t *rsc, const char *requested_name,
     /*  */
2316                      const char *rsc_action, GHashTable *override_hash,
2317                      guint timeout_ms, cib_t *cib, pcmk_scheduler_t *scheduler,
2318                      int resource_verbose, gboolean force, int check_level)
2319 {
2320     pcmk__output_t *out = scheduler->priv->out;
2321     crm_exit_t exit_code = CRM_EX_OK;
2322     const char *rid = requested_name;
2323     const char *rtype = NULL;
2324     const char *rprov = NULL;
2325     const char *rclass = NULL;
2326     GHashTable *params = NULL;
2327 
2328     if (pcmk__strcase_any_of(rsc_action, "force-start", "force-demote",
2329                                     "force-promote", NULL)) {
2330         if (pcmk__is_clone(rsc)) {
2331             GList *nodes = cli_resource_search(rsc, requested_name, scheduler);
2332             if(nodes != NULL && force == FALSE) {
2333                 out->err(out, "It is not safe to %s %s here: the cluster claims it is already active",
2334                          rsc_action, rsc->id);
2335                 out->err(out,
2336                          "Try setting "
2337                          PCMK_META_TARGET_ROLE "=" PCMK_ROLE_STOPPED
2338                          " first or specifying the force option");
2339                 return CRM_EX_UNSAFE;
2340             }
2341 
2342             g_list_free_full(nodes, free);
2343         }
2344     }
2345 
2346     if (pcmk__is_clone(rsc)) {
2347         /* Grab the first child resource in the hope it's not a group */
2348         rsc = rsc->priv->children->data;
2349     }
2350 
2351     if (pcmk__is_group(rsc)) {
2352         out->err(out, "Sorry, the %s option doesn't support group resources", rsc_action);
2353         return CRM_EX_UNIMPLEMENT_FEATURE;
2354     } else if (pcmk__is_bundled(rsc)) {
2355         out->err(out, "Sorry, the %s option doesn't support bundled resources", rsc_action);
2356         return CRM_EX_UNIMPLEMENT_FEATURE;
2357     }
2358 
2359     rclass = crm_element_value(rsc->priv->xml, PCMK_XA_CLASS);
2360     rprov = crm_element_value(rsc->priv->xml, PCMK_XA_PROVIDER);
2361     rtype = crm_element_value(rsc->priv->xml, PCMK_XA_TYPE);
2362 
2363     params = generate_resource_params(rsc, NULL /* @TODO use local node */,
2364                                       scheduler);
2365 
2366     if (timeout_ms == 0U) {
2367         timeout_ms = get_action_timeout(rsc, get_action(rsc_action));
2368     }
2369 
2370     if (!pcmk__is_anonymous_clone(rsc->priv->parent)) {
2371         rid = rsc->id;
2372     }
2373 
2374     exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype, rsc_action,
2375                                                  params, override_hash, timeout_ms,
2376                                                  resource_verbose, force, check_level);
2377     return exit_code;
2378 }
2379 
2380 // \return Standard Pacemaker return code
2381 int
2382 cli_resource_move(const pcmk_resource_t *rsc, const char *rsc_id,
     /*  */
2383                   const char *host_name, const char *move_lifetime, cib_t *cib,
2384                   pcmk_scheduler_t *scheduler, gboolean promoted_role_only,
2385                   gboolean force)
2386 {
2387     pcmk__output_t *out = scheduler->priv->out;
2388     int rc = pcmk_rc_ok;
2389     unsigned int count = 0;
2390     pcmk_node_t *current = NULL;
2391     pcmk_node_t *dest = pcmk_find_node(scheduler, host_name);
2392     bool cur_is_dest = false;
2393 
2394     if (dest == NULL) {
2395         return pcmk_rc_node_unknown;
2396     }
2397 
2398     if (promoted_role_only
2399         && !pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
2400 
2401         const pcmk_resource_t *p = pe__const_top_resource(rsc, false);
2402 
2403         if (pcmk_is_set(p->flags, pcmk__rsc_promotable)) {
2404             out->info(out, "Using parent '%s' for move instead of '%s'.", rsc->id, rsc_id);
2405             rsc_id = p->id;
2406             rsc = p;
2407 
2408         } else {
2409             out->info(out, "Ignoring --promoted option: %s is not promotable",
2410                       rsc_id);
2411             promoted_role_only = FALSE;
2412         }
2413     }
2414 
2415     current = pe__find_active_requires(rsc, &count);
2416 
2417     if (pcmk_is_set(rsc->flags, pcmk__rsc_promotable)) {
2418         unsigned int promoted_count = 0;
2419         pcmk_node_t *promoted_node = NULL;
2420 
2421         for (const GList *iter = rsc->priv->children;
2422              iter != NULL; iter = iter->next) {
2423 
2424             const pcmk_resource_t *child = (const pcmk_resource_t *) iter->data;
2425             enum rsc_role_e child_role = child->priv->fns->state(child, TRUE);
2426 
2427             if (child_role == pcmk_role_promoted) {
2428                 rsc = child;
2429                 promoted_node = pcmk__current_node(child);
2430                 promoted_count++;
2431             }
2432         }
2433         if (promoted_role_only || (promoted_count != 0)) {
2434             count = promoted_count;
2435             current = promoted_node;
2436         }
2437 
2438     }
2439 
2440     if (count > 1) {
2441         if (pcmk__is_clone(rsc)) {
2442             current = NULL;
2443         } else {
2444             return pcmk_rc_multiple;
2445         }
2446     }
2447 
2448     if (pcmk__same_node(current, dest)) {
2449         cur_is_dest = true;
2450         if (force) {
2451             crm_info("%s is already %s on %s, reinforcing placement with location constraint.",
2452                      rsc_id, promoted_role_only?"promoted":"active",
2453                      pcmk__node_name(dest));
2454         } else {
2455             return pcmk_rc_already;
2456         }
2457     }
2458 
2459     /* Clear any previous prefer constraints across all nodes. */
2460     cli_resource_clear(rsc_id, NULL, scheduler->nodes, cib, false, force);
2461 
2462     /* Clear any previous ban constraints on 'dest'. */
2463     cli_resource_clear(rsc_id, dest->priv->name, scheduler->nodes, cib, true,
2464                        force);
2465 
2466     /* Record an explicit preference for 'dest' */
2467     rc = cli_resource_prefer(out, rsc_id, dest->priv->name, move_lifetime,
2468                              cib, promoted_role_only, PCMK_ROLE_PROMOTED);
2469 
2470     crm_trace("%s%s now prefers %s%s",
2471               rsc->id, (promoted_role_only? " (promoted)" : ""),
2472               pcmk__node_name(dest), force?"(forced)":"");
2473 
2474     /* only ban the previous location if current location != destination location.
2475      * it is possible to use -M to enforce a location without regard of where the
2476      * resource is currently located */
2477     if (force && !cur_is_dest) {
2478         /* Ban the original location if possible */
2479         if(current) {
2480             (void)cli_resource_ban(out, rsc_id, current->priv->name,
2481                                    move_lifetime, cib, promoted_role_only,
2482                                    PCMK_ROLE_PROMOTED);
2483         } else if(count > 1) {
2484             out->info(out, "Resource '%s' is currently %s in %d locations. "
2485                       "One may now move to %s",
2486                       rsc_id, (promoted_role_only? "promoted" : "active"),
2487                       count, pcmk__node_name(dest));
2488             out->info(out, "To prevent '%s' from being %s at a specific location, "
2489                       "specify a node.",
2490                       rsc_id, (promoted_role_only? "promoted" : "active"));
2491 
2492         } else {
2493             crm_trace("Not banning %s from its current location: not active", rsc_id);
2494         }
2495     }
2496 
2497     return rc;
2498 }