daemons/controld/controld

/* */
This source file includes following definitions.
lrm_connection_destroy
make_stop_id
copy_instance_keys
copy_meta_keys
history_remove_recurring_op
history_free_recurring_ops
history_free
update_history_cache
send_task_ok_ack
op_node_name
lrm_op_callback
try_local_executor_connect
do_lrm_control
lrm_state_verify_stopped
is_rsc_active
build_active_RAs
controld_query_executor_state
controld_rc2event
controld_trigger_delete_refresh
notify_deleted
lrm_remove_deleted_rsc
lrm_remove_deleted_op
delete_rsc_entry
last_failed_matches_op
lrm_clear_last_failure
cancel_op
cancel_action_by_key
cancel_op_key
get_lrm_resource
delete_resource
get_fake_call_id
fake_op_status
force_reprobe
synthesize_lrmd_failure
lrm_op_target
fail_lrm_resource
handle_reprobe_op
do_lrm_cancel
do_lrm_delete
new_metadata_cb_data
free_metadata_cb_data
metadata_complete
do_lrm_invoke
construct_op
controld_ack_event_directly
verify_stopped
stop_recurring_action_by_rsc
stop_recurring_actions
should_cancel_recurring
should_nack_action
do_lrm_rsc_op
do_lrm_event
unescape_newlines
did_lrm_rsc_op_fail
log_executor_event
process_lrm_event
   1 /*
   2  * Copyright 2004-2024 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <regex.h>
  13 #include <sys/param.h>
  14 #include <sys/types.h>
  15 #include <sys/wait.h>
  16 
  17 #include <crm/crm.h>
  18 #include <crm/lrmd.h>           // lrmd_event_data_t, lrmd_rsc_info_t, etc.
  19 #include <crm/services.h>
  20 #include <crm/common/xml.h>
  21 #include <crm/pengine/rules.h>
  22 #include <crm/lrmd_internal.h>
  23 
  24 #include <pacemaker-internal.h>
  25 #include <pacemaker-controld.h>
  26 
  27 #define START_DELAY_THRESHOLD 5 * 60 * 1000
  28 #define MAX_LRM_REG_FAILS 30
  29 
  30 struct delete_event_s {
  31     int rc;
  32     const char *rsc;
  33     lrm_state_t *lrm_state;
  34 };
  35 
  36 static gboolean is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id);
  37 static gboolean build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list);
  38 static gboolean stop_recurring_actions(gpointer key, gpointer value, gpointer user_data);
  39 
  40 static lrmd_event_data_t *construct_op(const lrm_state_t *lrm_state,
  41                                        const xmlNode *rsc_op,
  42                                        const char *rsc_id,
  43                                        const char *operation);
  44 static void do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc,
  45                           xmlNode *msg, struct ra_metadata_s *md);
  46 
  47 static gboolean lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state,
  48                                          int log_level);
  49 
  50 static void
  51 lrm_connection_destroy(void)
     /*  */
  52 {
  53     if (pcmk_is_set(controld_globals.fsa_input_register, R_LRM_CONNECTED)) {
  54         crm_crit("Lost connection to local executor");
  55         register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);
  56         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
  57     }
  58 }
  59 
  60 static char *
  61 make_stop_id(const char *rsc, int call_id)
     /*  */
  62 {
  63     return crm_strdup_printf("%s:%d", rsc, call_id);
  64 }
  65 
  66 static void
  67 copy_instance_keys(gpointer key, gpointer value, gpointer user_data)
     /*  */
  68 {
  69     if (strstr(key, CRM_META "_") == NULL) {
  70         pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
  71     }
  72 }
  73 
  74 static void
  75 copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
     /*  */
  76 {
  77     if (strstr(key, CRM_META "_") != NULL) {
  78         pcmk__insert_dup(user_data, (const char *) key, (const char *) value);
  79     }
  80 }
  81 
  82 /*!
  83  * \internal
  84  * \brief Remove a recurring operation from a resource's history
  85  *
  86  * \param[in,out] history  Resource history to modify
  87  * \param[in]     op       Operation to remove
  88  *
  89  * \return TRUE if the operation was found and removed, FALSE otherwise
  90  */
  91 static gboolean
  92 history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
     /*  */
  93 {
  94     GList *iter;
  95 
  96     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
  97         lrmd_event_data_t *existing = iter->data;
  98 
  99         if ((op->interval_ms == existing->interval_ms)
 100             && pcmk__str_eq(op->rsc_id, existing->rsc_id, pcmk__str_none)
 101             && pcmk__str_eq(op->op_type, existing->op_type, pcmk__str_casei)) {
 102 
 103             history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
 104             lrmd_free_event(existing);
 105             return TRUE;
 106         }
 107     }
 108     return FALSE;
 109 }
 110 
 111 /*!
 112  * \internal
 113  * \brief Free all recurring operations in resource history
 114  *
 115  * \param[in,out] history  Resource history to modify
 116  */
 117 static void
 118 history_free_recurring_ops(rsc_history_t *history)
     /*  */
 119 {
 120     GList *iter;
 121 
 122     for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
 123         lrmd_free_event(iter->data);
 124     }
 125     g_list_free(history->recurring_op_list);
 126     history->recurring_op_list = NULL;
 127 }
 128 
 129 /*!
 130  * \internal
 131  * \brief Free resource history
 132  *
 133  * \param[in,out] history  Resource history to free
 134  */
 135 void
 136 history_free(gpointer data)
     /*  */
 137 {
 138     rsc_history_t *history = (rsc_history_t*)data;
 139 
 140     if (history->stop_params) {
 141         g_hash_table_destroy(history->stop_params);
 142     }
 143 
 144     /* Don't need to free history->rsc.id because it's set to history->id */
 145     free(history->rsc.type);
 146     free(history->rsc.standard);
 147     free(history->rsc.provider);
 148 
 149     lrmd_free_event(history->failed);
 150     lrmd_free_event(history->last);
 151     free(history->id);
 152     history_free_recurring_ops(history);
 153     free(history);
 154 }
 155 
 156 static void
 157 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
     /*  */
 158 {
 159     int target_rc = 0;
 160     rsc_history_t *entry = NULL;
 161 
 162     if (op->rsc_deleted) {
 163         crm_debug("Purged history for '%s' after %s", op->rsc_id, op->op_type);
 164         controld_delete_resource_history(op->rsc_id, lrm_state->node_name,
 165                                          NULL, crmd_cib_smart_opt());
 166         return;
 167     }
 168 
 169     if (pcmk__str_eq(op->op_type, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
 170         return;
 171     }
 172 
 173     crm_debug("Updating history for '%s' with %s op", op->rsc_id, op->op_type);
 174 
 175     entry = g_hash_table_lookup(lrm_state->resource_history, op->rsc_id);
 176     if (entry == NULL && rsc) {
 177         entry = pcmk__assert_alloc(1, sizeof(rsc_history_t));
 178         entry->id = pcmk__str_copy(op->rsc_id);
 179         g_hash_table_insert(lrm_state->resource_history, entry->id, entry);
 180 
 181         entry->rsc.id = entry->id;
 182         entry->rsc.type = pcmk__str_copy(rsc->type);
 183         entry->rsc.standard = pcmk__str_copy(rsc->standard);
 184         entry->rsc.provider = pcmk__str_copy(rsc->provider);
 185 
 186     } else if (entry == NULL) {
 187         crm_info("Resource %s no longer exists, not updating cache", op->rsc_id);
 188         return;
 189     }
 190 
 191     entry->last_callid = op->call_id;
 192     target_rc = rsc_op_expected_rc(op);
 193     if (op->op_status == PCMK_EXEC_CANCELLED) {
 194         if (op->interval_ms > 0) {
 195             crm_trace("Removing cancelled recurring op: " PCMK__OP_FMT,
 196                       op->rsc_id, op->op_type, op->interval_ms);
 197             history_remove_recurring_op(entry, op);
 198             return;
 199         } else {
 200             crm_trace("Skipping " PCMK__OP_FMT " rc=%d, status=%d",
 201                       op->rsc_id, op->op_type, op->interval_ms, op->rc,
 202                       op->op_status);
 203         }
 204 
 205     } else if (did_rsc_op_fail(op, target_rc)) {
 206         /* Store failed monitors here, otherwise the block below will cause them
 207          * to be forgotten when a stop happens.
 208          */
 209         if (entry->failed) {
 210             lrmd_free_event(entry->failed);
 211         }
 212         entry->failed = lrmd_copy_event(op);
 213 
 214     } else if (op->interval_ms == 0) {
 215         if (entry->last) {
 216             lrmd_free_event(entry->last);
 217         }
 218         entry->last = lrmd_copy_event(op);
 219 
 220         if (op->params && pcmk__strcase_any_of(op->op_type, PCMK_ACTION_START,
 221                                                PCMK_ACTION_RELOAD,
 222                                                PCMK_ACTION_RELOAD_AGENT,
 223                                                PCMK_ACTION_MONITOR, NULL)) {
 224             if (entry->stop_params) {
 225                 g_hash_table_destroy(entry->stop_params);
 226             }
 227             entry->stop_params = pcmk__strkey_table(free, free);
 228 
 229             g_hash_table_foreach(op->params, copy_instance_keys, entry->stop_params);
 230         }
 231     }
 232 
 233     if (op->interval_ms > 0) {
 234         /* Ensure there are no duplicates */
 235         history_remove_recurring_op(entry, op);
 236 
 237         crm_trace("Adding recurring op: " PCMK__OP_FMT,
 238                   op->rsc_id, op->op_type, op->interval_ms);
 239         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
 240 
 241     } else if ((entry->recurring_op_list != NULL)
 242                 && !pcmk__str_eq(op->op_type, PCMK_ACTION_MONITOR,
 243                                  pcmk__str_casei)) {
 244         crm_trace("Dropping %d recurring ops because of: " PCMK__OP_FMT,
 245                   g_list_length(entry->recurring_op_list), op->rsc_id,
 246                   op->op_type, op->interval_ms);
 247         history_free_recurring_ops(entry);
 248     }
 249 }
 250 
 251 /*!
 252  * \internal
 253  * \brief Send a direct OK ack for a resource task
 254  *
 255  * \param[in] lrm_state  LRM connection
 256  * \param[in] input      Input message being ack'ed
 257  * \param[in] rsc_id     ID of affected resource
 258  * \param[in] rsc        Affected resource (if available)
 259  * \param[in] task       Operation task being ack'ed
 260  * \param[in] ack_host   Name of host to send ack to
 261  * \param[in] ack_sys    IPC system name to ack
 262  */
 263 static void
 264 send_task_ok_ack(const lrm_state_t *lrm_state, const ha_msg_input_t *input,
     /*  */
 265                  const char *rsc_id, const lrmd_rsc_info_t *rsc,
 266                  const char *task, const char *ack_host, const char *ack_sys)
 267 {
 268     lrmd_event_data_t *op = construct_op(lrm_state, input->xml, rsc_id, task);
 269 
 270     lrmd__set_result(op, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 271     controld_ack_event_directly(ack_host, ack_sys, rsc, op, rsc_id);
 272     lrmd_free_event(op);
 273 }
 274 
 275 static inline const char *
 276 op_node_name(lrmd_event_data_t *op)
     /*  */
 277 {
 278     return pcmk__s(op->remote_nodename,
 279                    controld_globals.cluster->priv->node_name);
 280 }
 281 
 282 void
 283 lrm_op_callback(lrmd_event_data_t * op)
     /*  */
 284 {
 285     CRM_CHECK(op != NULL, return);
 286     switch (op->type) {
 287         case lrmd_event_disconnect:
 288             if (op->remote_nodename == NULL) {
 289                 /* If this is the local executor IPC connection, set the right
 290                  * bits in the controller when the connection goes down.
 291                  */
 292                 lrm_connection_destroy();
 293             }
 294             break;
 295 
 296         case lrmd_event_exec_complete:
 297             {
 298                 lrm_state_t *lrm_state =
 299                     controld_get_executor_state(op_node_name(op), false);
 300 
 301                 pcmk__assert(lrm_state != NULL);
 302                 process_lrm_event(lrm_state, op, NULL, NULL);
 303             }
 304             break;
 305 
 306         default:
 307             break;
 308     }
 309 }
 310 
 311 static void
 312 try_local_executor_connect(long long action, fsa_data_t *msg_data,
     /*  */
 313                            lrm_state_t *lrm_state)
 314 {
 315     int rc = pcmk_rc_ok;
 316 
 317     crm_debug("Connecting to the local executor");
 318 
 319     // If we can connect, great
 320     rc = controld_connect_local_executor(lrm_state);
 321     if (rc == pcmk_rc_ok) {
 322         controld_set_fsa_input_flags(R_LRM_CONNECTED);
 323         crm_info("Connection to the local executor established");
 324         return;
 325     }
 326 
 327     // Otherwise, if we can try again, set a timer to do so
 328     if (lrm_state->num_lrm_register_fails < MAX_LRM_REG_FAILS) {
 329         crm_warn("Failed to connect to the local executor %d time%s "
 330                  "(%d max): %s", lrm_state->num_lrm_register_fails,
 331                  pcmk__plural_s(lrm_state->num_lrm_register_fails),
 332                  MAX_LRM_REG_FAILS, pcmk_rc_str(rc));
 333         controld_start_wait_timer();
 334         crmd_fsa_stall(FALSE);
 335         return;
 336     }
 337 
 338     // Otherwise give up
 339     crm_err("Failed to connect to the executor the max allowed "
 340             "%d time%s: %s", lrm_state->num_lrm_register_fails,
 341             pcmk__plural_s(lrm_state->num_lrm_register_fails),
 342             pcmk_rc_str(rc));
 343     register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 344 }
 345 
 346 /*       A_LRM_CONNECT  */
 347 void
 348 do_lrm_control(long long action,
     /*  */
 349                enum crmd_fsa_cause cause,
 350                enum crmd_fsa_state cur_state,
 351                enum crmd_fsa_input current_input, fsa_data_t * msg_data)
 352 {
 353     /* This only pertains to local executor connections. Remote connections are
 354      * handled as resources within the scheduler. Connecting and disconnecting
 355      * from remote executor instances is handled differently.
 356      */
 357 
 358     lrm_state_t *lrm_state = NULL;
 359 
 360     if (controld_globals.cluster->priv->node_name == NULL) {
 361         return; // Shouldn't be possible
 362     }
 363     lrm_state = controld_get_executor_state(NULL, true);
 364     if (lrm_state == NULL) {
 365         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
 366         return;
 367     }
 368 
 369     if (action & A_LRM_DISCONNECT) {
 370         if (lrm_state_verify_stopped(lrm_state, cur_state, LOG_INFO) == FALSE) {
 371             if (action == A_LRM_DISCONNECT) {
 372                 crmd_fsa_stall(FALSE);
 373                 return;
 374             }
 375         }
 376 
 377         controld_clear_fsa_input_flags(R_LRM_CONNECTED);
 378         lrm_state_disconnect(lrm_state);
 379         lrm_state_reset_tables(lrm_state, FALSE);
 380     }
 381 
 382     if (action & A_LRM_CONNECT) {
 383         try_local_executor_connect(action, msg_data, lrm_state);
 384     }
 385 
 386     if (action & ~(A_LRM_CONNECT | A_LRM_DISCONNECT)) {
 387         crm_err("Unexpected action %s in %s", fsa_action2string(action),
 388                 __func__);
 389     }
 390 }
 391 
 392 static gboolean
 393 lrm_state_verify_stopped(lrm_state_t * lrm_state, enum crmd_fsa_state cur_state, int log_level)
     /*  */
 394 {
 395     int counter = 0;
 396     gboolean rc = TRUE;
 397     const char *when = "lrm disconnect";
 398 
 399     GHashTableIter gIter;
 400     const char *key = NULL;
 401     rsc_history_t *entry = NULL;
 402     active_op_t *pending = NULL;
 403 
 404     crm_debug("Checking for active resources before exit");
 405 
 406     if (cur_state == S_TERMINATE) {
 407         log_level = LOG_ERR;
 408         when = "shutdown";
 409 
 410     } else if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 411         when = "shutdown... waiting";
 412     }
 413 
 414     if ((lrm_state->active_ops != NULL) && lrm_state_is_connected(lrm_state)) {
 415         guint removed = g_hash_table_foreach_remove(lrm_state->active_ops,
 416                                                     stop_recurring_actions,
 417                                                     lrm_state);
 418         guint nremaining = g_hash_table_size(lrm_state->active_ops);
 419 
 420         if (removed || nremaining) {
 421             crm_notice("Stopped %u recurring operation%s at %s (%u remaining)",
 422                        removed, pcmk__plural_s(removed), when, nremaining);
 423         }
 424     }
 425 
 426     if (lrm_state->active_ops != NULL) {
 427         g_hash_table_iter_init(&gIter, lrm_state->active_ops);
 428         while (g_hash_table_iter_next(&gIter, NULL, (void **)&pending)) {
 429             /* Ignore recurring actions in the shutdown calculations */
 430             if (pending->interval_ms == 0) {
 431                 counter++;
 432             }
 433         }
 434     }
 435 
 436     if (counter > 0) {
 437         do_crm_log(log_level, "%d pending executor operation%s at %s",
 438                    counter, pcmk__plural_s(counter), when);
 439 
 440         if ((cur_state == S_TERMINATE)
 441             || !pcmk_is_set(controld_globals.fsa_input_register,
 442                             R_SENT_RSC_STOP)) {
 443             g_hash_table_iter_init(&gIter, lrm_state->active_ops);
 444             while (g_hash_table_iter_next(&gIter, (gpointer*)&key, (gpointer*)&pending)) {
 445                 do_crm_log(log_level, "Pending action: %s (%s)", key, pending->op_key);
 446             }
 447 
 448         } else {
 449             rc = FALSE;
 450         }
 451         return rc;
 452     }
 453 
 454     if (lrm_state->resource_history == NULL) {
 455         return rc;
 456     }
 457 
 458     if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)) {
 459         /* At this point we're not waiting, we're just shutting down */
 460         when = "shutdown";
 461     }
 462 
 463     counter = 0;
 464     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
 465     while (g_hash_table_iter_next(&gIter, NULL, (gpointer*)&entry)) {
 466         if (is_rsc_active(lrm_state, entry->id) == FALSE) {
 467             continue;
 468         }
 469 
 470         counter++;
 471         if (log_level == LOG_ERR) {
 472             crm_info("Found %s active at %s", entry->id, when);
 473         } else {
 474             crm_trace("Found %s active at %s", entry->id, when);
 475         }
 476         if (lrm_state->active_ops != NULL) {
 477             GHashTableIter hIter;
 478 
 479             g_hash_table_iter_init(&hIter, lrm_state->active_ops);
 480             while (g_hash_table_iter_next(&hIter, (gpointer*)&key, (gpointer*)&pending)) {
 481                 if (pcmk__str_eq(entry->id, pending->rsc_id, pcmk__str_none)) {
 482                     crm_notice("%sction %s (%s) incomplete at %s",
 483                                pending->interval_ms == 0 ? "A" : "Recurring a",
 484                                key, pending->op_key, when);
 485                 }
 486             }
 487         }
 488     }
 489 
 490     if (counter) {
 491         crm_err("%d resource%s active at %s",
 492                 counter, (counter == 1)? " was" : "s were", when);
 493     }
 494 
 495     return rc;
 496 }
 497 
 498 static gboolean
 499 is_rsc_active(lrm_state_t * lrm_state, const char *rsc_id)
     /*  */
 500 {
 501     rsc_history_t *entry = NULL;
 502 
 503     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
 504     if (entry == NULL || entry->last == NULL) {
 505         return FALSE;
 506     }
 507 
 508     crm_trace("Processing %s: %s.%d=%d", rsc_id, entry->last->op_type,
 509               entry->last->interval_ms, entry->last->rc);
 510     if ((entry->last->rc == PCMK_OCF_OK)
 511         && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_STOP,
 512                         pcmk__str_casei)) {
 513         return FALSE;
 514 
 515     } else if (entry->last->rc == PCMK_OCF_OK
 516                && pcmk__str_eq(entry->last->op_type, PCMK_ACTION_MIGRATE_TO,
 517                                pcmk__str_casei)) {
 518         // A stricter check is too complex ... leave that to the scheduler
 519         return FALSE;
 520 
 521     } else if (entry->last->rc == PCMK_OCF_NOT_RUNNING) {
 522         return FALSE;
 523 
 524     } else if ((entry->last->interval_ms == 0)
 525                && (entry->last->rc == PCMK_OCF_NOT_CONFIGURED)) {
 526         /* Badly configured resources can't be reliably stopped */
 527         return FALSE;
 528     }
 529 
 530     return TRUE;
 531 }
 532 
 533 static gboolean
 534 build_active_RAs(lrm_state_t * lrm_state, xmlNode * rsc_list)
     /*  */
 535 {
 536     GHashTableIter iter;
 537     rsc_history_t *entry = NULL;
 538 
 539     g_hash_table_iter_init(&iter, lrm_state->resource_history);
 540     while (g_hash_table_iter_next(&iter, NULL, (void **)&entry)) {
 541 
 542         GList *gIter = NULL;
 543         xmlNode *xml_rsc = pcmk__xe_create(rsc_list, PCMK__XE_LRM_RESOURCE);
 544 
 545         crm_xml_add(xml_rsc, PCMK_XA_ID, entry->id);
 546         crm_xml_add(xml_rsc, PCMK_XA_TYPE, entry->rsc.type);
 547         crm_xml_add(xml_rsc, PCMK_XA_CLASS, entry->rsc.standard);
 548         crm_xml_add(xml_rsc, PCMK_XA_PROVIDER, entry->rsc.provider);
 549 
 550         if (entry->last && entry->last->params) {
 551             static const char *name = CRM_META "_" PCMK__META_CONTAINER;
 552             const char *container = g_hash_table_lookup(entry->last->params,
 553                                                         name);
 554 
 555             if (container) {
 556                 crm_trace("Resource %s is a part of container resource %s", entry->id, container);
 557                 crm_xml_add(xml_rsc, PCMK__META_CONTAINER, container);
 558             }
 559         }
 560         controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->failed,
 561                                           lrm_state->node_name);
 562         controld_add_resource_history_xml(xml_rsc, &(entry->rsc), entry->last,
 563                                           lrm_state->node_name);
 564         for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIter->next) {
 565             controld_add_resource_history_xml(xml_rsc, &(entry->rsc), gIter->data,
 566                                               lrm_state->node_name);
 567         }
 568     }
 569 
 570     return FALSE;
 571 }
 572 
 573 xmlNode *
 574 controld_query_executor_state(void)
     /*  */
 575 {
 576     xmlNode *xml_state = NULL;
 577     xmlNode *xml_data = NULL;
 578     xmlNode *rsc_list = NULL;
 579     pcmk__node_status_t *peer = NULL;
 580     lrm_state_t *lrm_state = controld_get_executor_state(NULL, false);
 581 
 582     if (!lrm_state) {
 583         crm_err("Could not get executor state for local node");
 584         return NULL;
 585     }
 586 
 587     peer = pcmk__get_node(0, lrm_state->node_name, NULL, pcmk__node_search_any);
 588     CRM_CHECK(peer != NULL, return NULL);
 589 
 590     xml_state = create_node_state_update(peer,
 591                                          node_update_cluster|node_update_peer,
 592                                          NULL, __func__);
 593     if (xml_state == NULL) {
 594         return NULL;
 595     }
 596 
 597     xml_data = pcmk__xe_create(xml_state, PCMK__XE_LRM);
 598     crm_xml_add(xml_data, PCMK_XA_ID, peer->xml_id);
 599     rsc_list = pcmk__xe_create(xml_data, PCMK__XE_LRM_RESOURCES);
 600 
 601     /* Build a list of active (not always running) resources */
 602     build_active_RAs(lrm_state, rsc_list);
 603 
 604     crm_log_xml_trace(xml_state, "Current executor state");
 605 
 606     return xml_state;
 607 }
 608 
 609 /*!
 610  * \internal
 611  * \brief Map standard Pacemaker return code to operation status and OCF code
 612  *
 613  * \param[out] event  Executor event whose status and return code should be set
 614  * \param[in]  rc     Standard Pacemaker return code
 615  */
 616 void
 617 controld_rc2event(lrmd_event_data_t *event, int rc)
     /*  */
 618 {
 619     /* This is called for cleanup requests from controller peers/clients, not
 620      * for resource actions, so no exit reason is needed.
 621      */
 622     switch (rc) {
 623         case pcmk_rc_ok:
 624             lrmd__set_result(event, PCMK_OCF_OK, PCMK_EXEC_DONE, NULL);
 625             break;
 626         case EACCES:
 627             lrmd__set_result(event, PCMK_OCF_INSUFFICIENT_PRIV,
 628                              PCMK_EXEC_ERROR, NULL);
 629             break;
 630         default:
 631             lrmd__set_result(event, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_ERROR,
 632                              NULL);
 633             break;
 634     }
 635 }
 636 
 637 /*!
 638  * \internal
 639  * \brief Trigger a new transition after CIB status was deleted
 640  *
 641  * If a CIB status delete was not expected (as part of the transition graph),
 642  * trigger a new transition by updating the (arbitrary) "last-lrm-refresh"
 643  * cluster property.
 644  *
 645  * \param[in] from_sys  IPC name that requested the delete
 646  * \param[in] rsc_id    Resource whose status was deleted (for logging only)
 647  */
 648 void
 649 controld_trigger_delete_refresh(const char *from_sys, const char *rsc_id)
     /*  */
 650 {
 651     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_casei)) {
 652         char *now_s = crm_strdup_printf("%lld", (long long) time(NULL));
 653 
 654         crm_debug("Triggering a refresh after %s cleaned %s", from_sys, rsc_id);
 655         cib__update_node_attr(controld_globals.logger_out,
 656                               controld_globals.cib_conn, cib_none,
 657                               PCMK_XE_CRM_CONFIG, NULL, NULL, NULL, NULL,
 658                               "last-lrm-refresh", now_s, NULL, NULL);
 659         free(now_s);
 660     }
 661 }
 662 
 663 static void
 664 notify_deleted(lrm_state_t * lrm_state, ha_msg_input_t * input, const char *rsc_id, int rc)
     /*  */
 665 {
 666     lrmd_event_data_t *op = NULL;
 667     const char *from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
 668     const char *from_host = crm_element_value(input->msg, PCMK__XA_SRC);
 669 
 670     crm_info("Notifying %s on %s that %s was%s deleted",
 671              from_sys, (from_host? from_host : "localhost"), rsc_id,
 672              ((rc == pcmk_ok)? "" : " not"));
 673     op = construct_op(lrm_state, input->xml, rsc_id, PCMK_ACTION_DELETE);
 674     controld_rc2event(op, pcmk_legacy2rc(rc));
 675     controld_ack_event_directly(from_host, from_sys, NULL, op, rsc_id);
 676     lrmd_free_event(op);
 677     controld_trigger_delete_refresh(from_sys, rsc_id);
 678 }
 679 
 680 static gboolean
 681 lrm_remove_deleted_rsc(gpointer key, gpointer value, gpointer user_data)
     /*  */
 682 {
 683     struct delete_event_s *event = user_data;
 684     struct pending_deletion_op_s *op = value;
 685 
 686     if (pcmk__str_eq(event->rsc, op->rsc, pcmk__str_none)) {
 687         notify_deleted(event->lrm_state, op->input, event->rsc, event->rc);
 688         return TRUE;
 689     }
 690     return FALSE;
 691 }
 692 
 693 static gboolean
 694 lrm_remove_deleted_op(gpointer key, gpointer value, gpointer user_data)
     /*  */
 695 {
 696     const char *rsc = user_data;
 697     active_op_t *pending = value;
 698 
 699     if (pcmk__str_eq(rsc, pending->rsc_id, pcmk__str_none)) {
 700         crm_info("Removing op %s:%d for deleted resource %s",
 701                  pending->op_key, pending->call_id, rsc);
 702         return TRUE;
 703     }
 704     return FALSE;
 705 }
 706 
 707 static void
 708 delete_rsc_entry(lrm_state_t *lrm_state, ha_msg_input_t *input,
     /*  */
 709                  const char *rsc_id, GHashTableIter *rsc_iter, int rc,
 710                  const char *user_name, bool from_cib)
 711 {
 712     struct delete_event_s event;
 713 
 714     CRM_CHECK(rsc_id != NULL, return);
 715 
 716     if (rc == pcmk_ok) {
 717         char *rsc_id_copy = pcmk__str_copy(rsc_id);
 718 
 719         if (rsc_iter) {
 720             g_hash_table_iter_remove(rsc_iter);
 721         } else {
 722             g_hash_table_remove(lrm_state->resource_history, rsc_id_copy);
 723         }
 724 
 725         if (from_cib) {
 726             controld_delete_resource_history(rsc_id_copy, lrm_state->node_name,
 727                                              user_name, crmd_cib_smart_opt());
 728         }
 729         g_hash_table_foreach_remove(lrm_state->active_ops,
 730                                     lrm_remove_deleted_op, rsc_id_copy);
 731         free(rsc_id_copy);
 732     }
 733 
 734     if (input) {
 735         notify_deleted(lrm_state, input, rsc_id, rc);
 736     }
 737 
 738     event.rc = rc;
 739     event.rsc = rsc_id;
 740     event.lrm_state = lrm_state;
 741     g_hash_table_foreach_remove(lrm_state->deletion_ops, lrm_remove_deleted_rsc, &event);
 742 }
 743 
 744 static inline gboolean
 745 last_failed_matches_op(rsc_history_t *entry, const char *op, guint interval_ms)
     /*  */
 746 {
 747     if (entry == NULL) {
 748         return FALSE;
 749     }
 750     if (op == NULL) {
 751         return TRUE;
 752     }
 753     return (pcmk__str_eq(op, entry->failed->op_type, pcmk__str_casei)
 754             && (interval_ms == entry->failed->interval_ms));
 755 }
 756 
 757 /*!
 758  * \internal
 759  * \brief Clear a resource's last failure
 760  *
 761  * Erase a resource's last failure on a particular node from both the
 762  * LRM resource history in the CIB, and the resource history remembered
 763  * for the LRM state.
 764  *
 765  * \param[in] rsc_id      Resource name
 766  * \param[in] node_name   Node name
 767  * \param[in] operation   If specified, only clear if matching this operation
 768  * \param[in] interval_ms If operation is specified, it has this interval
 769  */
 770 void
 771 lrm_clear_last_failure(const char *rsc_id, const char *node_name,
     /*  */
 772                        const char *operation, guint interval_ms)
 773 {
 774     lrm_state_t *lrm_state = controld_get_executor_state(node_name, false);
 775 
 776     if (lrm_state == NULL) {
 777         return;
 778     }
 779     if (lrm_state->resource_history != NULL) {
 780         rsc_history_t *entry = g_hash_table_lookup(lrm_state->resource_history,
 781                                                    rsc_id);
 782 
 783         if (last_failed_matches_op(entry, operation, interval_ms)) {
 784             lrmd_free_event(entry->failed);
 785             entry->failed = NULL;
 786         }
 787     }
 788 }
 789 
 790 /* Returns: gboolean - cancellation is in progress */
 791 static gboolean
 792 cancel_op(lrm_state_t * lrm_state, const char *rsc_id, const char *key, int op, gboolean remove)
     /*  */
 793 {
 794     int rc = pcmk_ok;
 795     char *local_key = NULL;
 796     active_op_t *pending = NULL;
 797 
 798     CRM_CHECK(op != 0, return FALSE);
 799     CRM_CHECK(rsc_id != NULL, return FALSE);
 800     if (key == NULL) {
 801         local_key = make_stop_id(rsc_id, op);
 802         key = local_key;
 803     }
 804     pending = g_hash_table_lookup(lrm_state->active_ops, key);
 805 
 806     if (pending) {
 807         if (remove && !pcmk_is_set(pending->flags, active_op_remove)) {
 808             controld_set_active_op_flags(pending, active_op_remove);
 809             crm_debug("Scheduling %s for removal", key);
 810         }
 811 
 812         if (pcmk_is_set(pending->flags, active_op_cancelled)) {
 813             crm_debug("Operation %s already cancelled", key);
 814             free(local_key);
 815             return FALSE;
 816         }
 817         controld_set_active_op_flags(pending, active_op_cancelled);
 818 
 819     } else {
 820         crm_info("No pending op found for %s", key);
 821         free(local_key);
 822         return FALSE;
 823     }
 824 
 825     crm_debug("Cancelling op %d for %s (%s)", op, rsc_id, key);
 826     rc = lrm_state_cancel(lrm_state, pending->rsc_id, pending->op_type,
 827                           pending->interval_ms);
 828     if (rc == pcmk_ok) {
 829         crm_debug("Op %d for %s (%s): cancelled", op, rsc_id, key);
 830         free(local_key);
 831         return TRUE;
 832     }
 833 
 834     crm_debug("Op %d for %s (%s): Nothing to cancel", op, rsc_id, key);
 835     /* The caller needs to make sure the entry is
 836      * removed from the active operations list
 837      *
 838      * Usually by returning TRUE inside the worker function
 839      * supplied to g_hash_table_foreach_remove()
 840      *
 841      * Not removing the entry from active operations will block
 842      * the node from shutting down
 843      */
 844     free(local_key);
 845     return FALSE;
 846 }
 847 
 848 struct cancel_data {
 849     gboolean done;
 850     gboolean remove;
 851     const char *key;
 852     lrmd_rsc_info_t *rsc;
 853     lrm_state_t *lrm_state;
 854 };
 855 
 856 static gboolean
 857 cancel_action_by_key(gpointer key, gpointer value, gpointer user_data)
     /*  */
 858 {
 859     gboolean remove = FALSE;
 860     struct cancel_data *data = user_data;
 861     active_op_t *op = value;
 862 
 863     if (pcmk__str_eq(op->op_key, data->key, pcmk__str_none)) {
 864         data->done = TRUE;
 865         remove = !cancel_op(data->lrm_state, data->rsc->id, key, op->call_id, data->remove);
 866     }
 867     return remove;
 868 }
 869 
 870 static gboolean
 871 cancel_op_key(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, const char *key, gboolean remove)
     /*  */
 872 {
 873     guint removed = 0;
 874     struct cancel_data data;
 875 
 876     CRM_CHECK(rsc != NULL, return FALSE);
 877     CRM_CHECK(key != NULL, return FALSE);
 878 
 879     data.key = key;
 880     data.rsc = rsc;
 881     data.done = FALSE;
 882     data.remove = remove;
 883     data.lrm_state = lrm_state;
 884 
 885     removed = g_hash_table_foreach_remove(lrm_state->active_ops,
 886                                           cancel_action_by_key, &data);
 887     crm_trace("Removed %u op cache entries, new size: %u",
 888               removed, g_hash_table_size(lrm_state->active_ops));
 889     return data.done;
 890 }
 891 
 892 /*!
 893  * \internal
 894  * \brief Retrieve resource information from LRM
 895  *
 896  * \param[in,out]  lrm_state  Executor connection state to use
 897  * \param[in]      rsc_xml    XML containing resource configuration
 898  * \param[in]      do_create  If true, register resource if not already
 899  * \param[out]     rsc_info   Where to store information obtained from executor
 900  *
 901  * \retval pcmk_ok   Success (and rsc_info holds newly allocated result)
 902  * \retval -EINVAL   Required information is missing from arguments
 903  * \retval -ENOTCONN No active connection to LRM
 904  * \retval -ENODEV   Resource not found
 905  * \retval -errno    Error communicating with executor when registering resource
 906  *
 907  * \note Caller is responsible for freeing result on success.
 908  */
 909 static int
 910 get_lrm_resource(lrm_state_t *lrm_state, const xmlNode *rsc_xml,
     /*  */
 911                  gboolean do_create, lrmd_rsc_info_t **rsc_info)
 912 {
 913     const char *id = pcmk__xe_id(rsc_xml);
 914 
 915     CRM_CHECK(lrm_state && rsc_xml && rsc_info, return -EINVAL);
 916     CRM_CHECK(id, return -EINVAL);
 917 
 918     if (lrm_state_is_connected(lrm_state) == FALSE) {
 919         return -ENOTCONN;
 920     }
 921 
 922     crm_trace("Retrieving resource information for %s from the executor", id);
 923     *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 924 
 925     // If resource isn't known by ID, try clone name, if provided
 926     if (!*rsc_info) {
 927         const char *long_id = crm_element_value(rsc_xml, PCMK__XA_LONG_ID);
 928 
 929         if (long_id) {
 930             *rsc_info = lrm_state_get_rsc_info(lrm_state, long_id, 0);
 931         }
 932     }
 933 
 934     if ((*rsc_info == NULL) && do_create) {
 935         const char *class = crm_element_value(rsc_xml, PCMK_XA_CLASS);
 936         const char *provider = crm_element_value(rsc_xml, PCMK_XA_PROVIDER);
 937         const char *type = crm_element_value(rsc_xml, PCMK_XA_TYPE);
 938         int rc;
 939 
 940         crm_trace("Registering resource %s with the executor", id);
 941         rc = lrm_state_register_rsc(lrm_state, id, class, provider, type,
 942                                     lrmd_opt_drop_recurring);
 943         if (rc != pcmk_ok) {
 944             fsa_data_t *msg_data = NULL;
 945 
 946             crm_err("Could not register resource %s with the executor on %s: %s "
 947                     QB_XS " rc=%d",
 948                     id, lrm_state->node_name, pcmk_strerror(rc), rc);
 949 
 950             /* Register this as an internal error if this involves the local
 951              * executor. Otherwise, we're likely dealing with an unresponsive
 952              * remote node, which is not an FSA failure.
 953              */
 954             if (lrm_state_is_local(lrm_state) == TRUE) {
 955                 register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
 956             }
 957             return rc;
 958         }
 959 
 960         *rsc_info = lrm_state_get_rsc_info(lrm_state, id, 0);
 961     }
 962     return *rsc_info? pcmk_ok : -ENODEV;
 963 }
 964 
 965 static void
 966 delete_resource(lrm_state_t *lrm_state, const char *id, lrmd_rsc_info_t *rsc,
     /*  */
 967                 GHashTableIter *iter, const char *sys, const char *user,
 968                 ha_msg_input_t *request, bool unregister, bool from_cib)
 969 {
 970     int rc = pcmk_ok;
 971 
 972     crm_info("Removing resource %s from executor for %s%s%s",
 973              id, sys, (user? " as " : ""), (user? user : ""));
 974 
 975     if (rsc && unregister) {
 976         rc = lrm_state_unregister_rsc(lrm_state, id, 0);
 977     }
 978 
 979     if (rc == pcmk_ok) {
 980         crm_trace("Resource %s deleted from executor", id);
 981     } else if (rc == -EINPROGRESS) {
 982         crm_info("Deletion of resource '%s' from executor is pending", id);
 983         if (request) {
 984             struct pending_deletion_op_s *op = NULL;
 985             char *ref = crm_element_value_copy(request->msg, PCMK_XA_REFERENCE);
 986 
 987             op = pcmk__assert_alloc(1, sizeof(struct pending_deletion_op_s));
 988             op->rsc = pcmk__str_copy(rsc->id);
 989             op->input = copy_ha_msg_input(request);
 990             g_hash_table_insert(lrm_state->deletion_ops, ref, op);
 991         }
 992         return;
 993     } else {
 994         crm_warn("Could not delete '%s' from executor for %s%s%s: %s "
 995                  QB_XS " rc=%d", id, sys, (user? " as " : ""),
 996                  (user? user : ""), pcmk_strerror(rc), rc);
 997     }
 998 
 999     delete_rsc_entry(lrm_state, request, id, iter, rc, user, from_cib);
1000 }
1001 
1002 static int
1003 get_fake_call_id(lrm_state_t *lrm_state, const char *rsc_id)
     /*  */
1004 {
1005     int call_id = 999999999;
1006     rsc_history_t *entry = NULL;
1007 
1008     if(lrm_state) {
1009         entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1010     }
1011 
1012     /* Make sure the call id is greater than the last successful operation,
1013      * otherwise the failure will not result in a possible recovery of the resource
1014      * as it could appear the failure occurred before the successful start */
1015     if (entry) {
1016         call_id = entry->last_callid + 1;
1017     }
1018 
1019     if (call_id < 0) {
1020         call_id = 1;
1021     }
1022     return call_id;
1023 }
1024 
1025 static void
1026 fake_op_status(lrm_state_t *lrm_state, lrmd_event_data_t *op, int op_status,
     /*  */
1027                enum ocf_exitcode op_exitcode, const char *exit_reason)
1028 {
1029     op->call_id = get_fake_call_id(lrm_state, op->rsc_id);
1030     op->t_run = time(NULL);
1031     op->t_rcchange = op->t_run;
1032     lrmd__set_result(op, op_exitcode, op_status, exit_reason);
1033 }
1034 
1035 static void
1036 force_reprobe(lrm_state_t *lrm_state, const char *from_sys,
     /*  */
1037               const char *from_host, const char *user_name,
1038               gboolean is_remote_node, bool reprobe_all_nodes)
1039 {
1040     GHashTableIter gIter;
1041     rsc_history_t *entry = NULL;
1042 
1043     crm_info("Clearing resource history on node %s", lrm_state->node_name);
1044     g_hash_table_iter_init(&gIter, lrm_state->resource_history);
1045     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
1046         /* only unregister the resource during a reprobe if it is not a remote connection
1047          * resource. otherwise unregistering the connection will terminate remote-node
1048          * membership */
1049         bool unregister = true;
1050 
1051         if (is_remote_lrmd_ra(NULL, NULL, entry->id)) {
1052             unregister = false;
1053 
1054             if (reprobe_all_nodes) {
1055                 lrm_state_t *remote_lrm_state =
1056                     controld_get_executor_state(entry->id, false);
1057 
1058                 if (remote_lrm_state != NULL) {
1059                     /* If reprobing all nodes, be sure to reprobe the remote
1060                      * node before clearing its connection resource
1061                      */
1062                     force_reprobe(remote_lrm_state, from_sys, from_host,
1063                                   user_name, TRUE, reprobe_all_nodes);
1064                 }
1065             }
1066         }
1067 
1068         /* Don't delete from the CIB, since we'll delete the whole node's LRM
1069          * state from the CIB soon
1070          */
1071         delete_resource(lrm_state, entry->id, &entry->rsc, &gIter, from_sys,
1072                         user_name, NULL, unregister, false);
1073     }
1074 
1075     /* Now delete the copy in the CIB */
1076     controld_delete_node_state(lrm_state->node_name, controld_section_lrm,
1077                                cib_none);
1078 }
1079 
1080 /*!
1081  * \internal
1082  * \brief Fail a requested action without actually executing it
1083  *
1084  * For an action that can't be executed, process it similarly to an actual
1085  * execution result, with specified error status (except for notify actions,
1086  * which will always be treated as successful).
1087  *
1088  * \param[in,out] lrm_state    Executor connection that action is for
1089  * \param[in]     action       Action XML from request
1090  * \param[in]     rc           Desired return code to use
1091  * \param[in]     op_status    Desired operation status to use
1092  * \param[in]     exit_reason  Human-friendly detail, if error
1093  */
1094 static void
1095 synthesize_lrmd_failure(lrm_state_t *lrm_state, const xmlNode *action,
     /*  */
1096                         int op_status, enum ocf_exitcode rc,
1097                         const char *exit_reason)
1098 {
1099     lrmd_event_data_t *op = NULL;
1100     const char *operation = crm_element_value(action, PCMK_XA_OPERATION);
1101     const char *target_node = crm_element_value(action, PCMK__META_ON_NODE);
1102     xmlNode *xml_rsc = pcmk__xe_first_child(action, PCMK_XE_PRIMITIVE, NULL,
1103                                             NULL);
1104 
1105     if ((xml_rsc == NULL) || (pcmk__xe_id(xml_rsc) == NULL)) {
1106         /* @TODO Should we do something else, like direct ack? */
1107         crm_info("Can't fake %s failure (%d) on %s without resource configuration",
1108                  crm_element_value(action, PCMK__XA_OPERATION_KEY), rc,
1109                  target_node);
1110         return;
1111 
1112     } else if(operation == NULL) {
1113         /* This probably came from crm_resource -C, nothing to do */
1114         crm_info("Can't fake %s failure (%d) on %s without operation",
1115                  pcmk__xe_id(xml_rsc), rc, target_node);
1116         return;
1117     }
1118 
1119     op = construct_op(lrm_state, action, pcmk__xe_id(xml_rsc), operation);
1120 
1121     if (pcmk__str_eq(operation, PCMK_ACTION_NOTIFY, pcmk__str_casei)) {
1122         // Notifications can't fail
1123         fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_OK, NULL);
1124     } else {
1125         fake_op_status(lrm_state, op, op_status, rc, exit_reason);
1126     }
1127 
1128     crm_info("Faking " PCMK__OP_FMT " result (%d) on %s",
1129              op->rsc_id, op->op_type, op->interval_ms, op->rc, target_node);
1130 
1131     // Process the result as if it came from the LRM
1132     process_lrm_event(lrm_state, op, NULL, action);
1133     lrmd_free_event(op);
1134 }
1135 
1136 /*!
1137  * \internal
1138  * \brief Get target of an LRM operation (replacing \p NULL with local node
1139  *        name)
1140  *
1141  * \param[in] xml  LRM operation data XML
1142  *
1143  * \return LRM operation target node name (local node or Pacemaker Remote node)
1144  */
1145 static const char *
1146 lrm_op_target(const xmlNode *xml)
     /*  */
1147 {
1148     const char *target = NULL;
1149 
1150     if (xml) {
1151         target = crm_element_value(xml, PCMK__META_ON_NODE);
1152     }
1153     if (target == NULL) {
1154         target = controld_globals.cluster->priv->node_name;
1155     }
1156     return target;
1157 }
1158 
1159 static void
1160 fail_lrm_resource(xmlNode *xml, lrm_state_t *lrm_state, const char *user_name,
     /*  */
1161                   const char *from_host, const char *from_sys)
1162 {
1163     lrmd_event_data_t *op = NULL;
1164     lrmd_rsc_info_t *rsc = NULL;
1165     xmlNode *xml_rsc = pcmk__xe_first_child(xml, PCMK_XE_PRIMITIVE, NULL, NULL);
1166 
1167     CRM_CHECK(xml_rsc != NULL, return);
1168 
1169     /* The executor simply executes operations and reports the results, without
1170      * any concept of success or failure, so to fail a resource, we must fake
1171      * what a failure looks like.
1172      *
1173      * To do this, we create a fake executor operation event for the resource,
1174      * and pass that event to the executor client callback so it will be
1175      * processed as if it came from the executor.
1176      */
1177     op = construct_op(lrm_state, xml, pcmk__xe_id(xml_rsc), "asyncmon");
1178 
1179     free((char*) op->user_data);
1180     op->user_data = NULL;
1181     op->interval_ms = 0;
1182 
1183     if (user_name && !pcmk__is_privileged(user_name)) {
1184         crm_err("%s does not have permission to fail %s",
1185                 user_name, pcmk__xe_id(xml_rsc));
1186         fake_op_status(lrm_state, op, PCMK_EXEC_ERROR,
1187                        PCMK_OCF_INSUFFICIENT_PRIV,
1188                        "Unprivileged user cannot fail resources");
1189         controld_ack_event_directly(from_host, from_sys, NULL, op,
1190                                     pcmk__xe_id(xml_rsc));
1191         lrmd_free_event(op);
1192         return;
1193     }
1194 
1195 
1196     if (get_lrm_resource(lrm_state, xml_rsc, TRUE, &rsc) == pcmk_ok) {
1197         crm_info("Failing resource %s...", rsc->id);
1198         fake_op_status(lrm_state, op, PCMK_EXEC_DONE, PCMK_OCF_UNKNOWN_ERROR,
1199                        "Simulated failure");
1200         process_lrm_event(lrm_state, op, NULL, xml);
1201         op->rc = PCMK_OCF_OK; // The request to fail the resource succeeded
1202         lrmd_free_rsc_info(rsc);
1203 
1204     } else {
1205         crm_info("Cannot find/create resource in order to fail it...");
1206         crm_log_xml_warn(xml, "bad input");
1207         fake_op_status(lrm_state, op, PCMK_EXEC_ERROR, PCMK_OCF_UNKNOWN_ERROR,
1208                        "Cannot fail unknown resource");
1209     }
1210 
1211     controld_ack_event_directly(from_host, from_sys, NULL, op,
1212                                 pcmk__xe_id(xml_rsc));
1213     lrmd_free_event(op);
1214 }
1215 
1216 static void
1217 handle_reprobe_op(lrm_state_t *lrm_state, xmlNode *msg, const char *from_sys,
     /*  */
1218                   const char *from_host, const char *user_name,
1219                   gboolean is_remote_node, bool reprobe_all_nodes)
1220 {
1221     crm_notice("Forcing the status of all resources to be redetected");
1222     force_reprobe(lrm_state, from_sys, from_host, user_name, is_remote_node,
1223                   reprobe_all_nodes);
1224 
1225     if (!pcmk__strcase_any_of(from_sys, CRM_SYSTEM_PENGINE, CRM_SYSTEM_TENGINE, NULL)) {
1226         xmlNode *reply = pcmk__new_reply(msg, NULL);
1227 
1228         crm_debug("ACK'ing re-probe from %s (%s)", from_sys, from_host);
1229 
1230         if (relay_message(reply, TRUE) == FALSE) {
1231             crm_log_xml_err(reply, "Unable to route reply");
1232         }
1233         pcmk__xml_free(reply);
1234     }
1235 }
1236 
1237 static bool do_lrm_cancel(ha_msg_input_t *input, lrm_state_t *lrm_state,
     /*  */
1238               lrmd_rsc_info_t *rsc, const char *from_host, const char *from_sys)
1239 {
1240     char *op_key = NULL;
1241     char *meta_key = NULL;
1242     int call = 0;
1243     const char *call_id = NULL;
1244     const char *op_task = NULL;
1245     guint interval_ms = 0;
1246     gboolean in_progress = FALSE;
1247     xmlNode *params = pcmk__xe_first_child(input->xml, PCMK__XE_ATTRIBUTES,
1248                                            NULL, NULL);
1249 
1250     CRM_CHECK(params != NULL, return FALSE);
1251 
1252     meta_key = crm_meta_name(PCMK_XA_OPERATION);
1253     op_task = crm_element_value(params, meta_key);
1254     free(meta_key);
1255     CRM_CHECK(op_task != NULL, return FALSE);
1256 
1257     meta_key = crm_meta_name(PCMK_META_INTERVAL);
1258     if (crm_element_value_ms(params, meta_key, &interval_ms) != pcmk_ok) {
1259         free(meta_key);
1260         return FALSE;
1261     }
1262     free(meta_key);
1263 
1264     op_key = pcmk__op_key(rsc->id, op_task, interval_ms);
1265 
1266     meta_key = crm_meta_name(PCMK__XA_CALL_ID);
1267     call_id = crm_element_value(params, meta_key);
1268     free(meta_key);
1269 
1270     crm_debug("Scheduler requested op %s (call=%s) be cancelled",
1271               op_key, (call_id? call_id : "NA"));
1272     pcmk__scan_min_int(call_id, &call, 0);
1273     if (call == 0) {
1274         // Normal case when the scheduler cancels a recurring op
1275         in_progress = cancel_op_key(lrm_state, rsc, op_key, TRUE);
1276 
1277     } else {
1278         // Normal case when the scheduler cancels an orphan op
1279         in_progress = cancel_op(lrm_state, rsc->id, NULL, call, TRUE);
1280     }
1281 
1282     // Acknowledge cancellation operation if for a remote connection resource
1283     if (!in_progress || is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1284         char *op_id = make_stop_id(rsc->id, call);
1285 
1286         if (is_remote_lrmd_ra(NULL, NULL, rsc->id) == FALSE) {
1287             crm_info("Nothing known about operation %d for %s", call, op_key);
1288         }
1289         controld_delete_action_history_by_key(rsc->id, lrm_state->node_name,
1290                                               op_key, call);
1291         send_task_ok_ack(lrm_state, input, rsc->id, rsc, op_task,
1292                          from_host, from_sys);
1293 
1294         /* needed at least for cancellation of a remote operation */
1295         if (lrm_state->active_ops != NULL) {
1296             g_hash_table_remove(lrm_state->active_ops, op_id);
1297         }
1298         free(op_id);
1299     }
1300 
1301     free(op_key);
1302     return TRUE;
1303 }
1304 
1305 static void
1306 do_lrm_delete(ha_msg_input_t *input, lrm_state_t *lrm_state,
     /*  */
1307               lrmd_rsc_info_t *rsc, const char *from_sys, const char *from_host,
1308               bool crm_rsc_delete, const char *user_name)
1309 {
1310     bool unregister = true;
1311     int cib_rc = controld_delete_resource_history(rsc->id, lrm_state->node_name,
1312                                                   user_name,
1313                                                   cib_dryrun|cib_sync_call);
1314 
1315     if (cib_rc != pcmk_rc_ok) {
1316         lrmd_event_data_t *op = NULL;
1317 
1318         op = construct_op(lrm_state, input->xml, rsc->id, PCMK_ACTION_DELETE);
1319 
1320         /* These are resource clean-ups, not actions, so no exit reason is
1321          * needed.
1322          */
1323         lrmd__set_result(op, pcmk_rc2ocf(cib_rc), PCMK_EXEC_ERROR, NULL);
1324         controld_ack_event_directly(from_host, from_sys, NULL, op, rsc->id);
1325         lrmd_free_event(op);
1326         return;
1327     }
1328 
1329     if (crm_rsc_delete && is_remote_lrmd_ra(NULL, NULL, rsc->id)) {
1330         unregister = false;
1331     }
1332 
1333     delete_resource(lrm_state, rsc->id, rsc, NULL, from_sys,
1334                     user_name, input, unregister, true);
1335 }
1336 
1337 // User data for asynchronous metadata execution
1338 struct metadata_cb_data {
1339     lrmd_rsc_info_t *rsc;   // Copy of resource information
1340     xmlNode *input_xml;     // Copy of FSA input XML
1341 };
1342 
1343 static struct metadata_cb_data *
1344 new_metadata_cb_data(lrmd_rsc_info_t *rsc, xmlNode *input_xml)
     /*  */
1345 {
1346     struct metadata_cb_data *data = NULL;
1347 
1348     data = pcmk__assert_alloc(1, sizeof(struct metadata_cb_data));
1349     data->input_xml = pcmk__xml_copy(NULL, input_xml);
1350     data->rsc = lrmd_copy_rsc_info(rsc);
1351     return data;
1352 }
1353 
1354 static void
1355 free_metadata_cb_data(struct metadata_cb_data *data)
     /*  */
1356 {
1357     lrmd_free_rsc_info(data->rsc);
1358     pcmk__xml_free(data->input_xml);
1359     free(data);
1360 }
1361 
1362 /*!
1363  * \internal
1364  * \brief Execute an action after metadata has been retrieved
1365  *
1366  * \param[in] pid        Ignored
1367  * \param[in] result     Result of metadata action
1368  * \param[in] user_data  Metadata callback data
1369  */
1370 static void
1371 metadata_complete(int pid, const pcmk__action_result_t *result, void *user_data)
     /*  */
1372 {
1373     struct metadata_cb_data *data = (struct metadata_cb_data *) user_data;
1374 
1375     struct ra_metadata_s *md = NULL;
1376     lrm_state_t *lrm_state =
1377         controld_get_executor_state(lrm_op_target(data->input_xml), false);
1378 
1379     if ((lrm_state != NULL) && pcmk__result_ok(result)) {
1380         md = controld_cache_metadata(lrm_state->metadata_cache, data->rsc,
1381                                      result->action_stdout);
1382     }
1383     if (!pcmk_is_set(controld_globals.fsa_input_register, R_HA_DISCONNECTED)) {
1384         do_lrm_rsc_op(lrm_state, data->rsc, data->input_xml, md);
1385     }
1386     free_metadata_cb_data(data);
1387 }
1388 
1389 /*       A_LRM_INVOKE   */
1390 void
1391 do_lrm_invoke(long long action,
     /*  */
1392               enum crmd_fsa_cause cause,
1393               enum crmd_fsa_state cur_state,
1394               enum crmd_fsa_input current_input, fsa_data_t * msg_data)
1395 {
1396     lrm_state_t *lrm_state = NULL;
1397     const char *crm_op = NULL;
1398     const char *from_sys = NULL;
1399     const char *from_host = NULL;
1400     const char *operation = NULL;
1401     ha_msg_input_t *input = fsa_typed_data(fsa_dt_ha_msg);
1402     const char *user_name = NULL;
1403     const char *target_node = lrm_op_target(input->xml);
1404     gboolean is_remote_node = FALSE;
1405     bool crm_rsc_delete = FALSE;
1406 
1407     // Message routed to the local node is targeting a specific, non-local node
1408     is_remote_node = !controld_is_local_node(target_node);
1409 
1410     lrm_state = controld_get_executor_state(target_node, false);
1411     if ((lrm_state == NULL) && is_remote_node) {
1412         crm_err("Failing action because local node has never had connection to remote node %s",
1413                 target_node);
1414         synthesize_lrmd_failure(NULL, input->xml, PCMK_EXEC_NOT_CONNECTED,
1415                                 PCMK_OCF_UNKNOWN_ERROR,
1416                                 "Local node has no connection to remote");
1417         return;
1418     }
1419     pcmk__assert(lrm_state != NULL);
1420 
1421     user_name = pcmk__update_acl_user(input->msg, PCMK__XA_CRM_USER, NULL);
1422     crm_op = crm_element_value(input->msg, PCMK__XA_CRM_TASK);
1423     from_sys = crm_element_value(input->msg, PCMK__XA_CRM_SYS_FROM);
1424     if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1425         from_host = crm_element_value(input->msg, PCMK__XA_SRC);
1426     }
1427 
1428     if (pcmk__str_eq(crm_op, PCMK_ACTION_LRM_DELETE, pcmk__str_none)) {
1429         if (!pcmk__str_eq(from_sys, CRM_SYSTEM_TENGINE, pcmk__str_none)) {
1430             crm_rsc_delete = TRUE; // from crm_resource
1431         }
1432         operation = PCMK_ACTION_DELETE;
1433 
1434     } else if (input->xml != NULL) {
1435         operation = crm_element_value(input->xml, PCMK_XA_OPERATION);
1436     }
1437 
1438     CRM_CHECK(!pcmk__str_empty(crm_op) || !pcmk__str_empty(operation), return);
1439 
1440     crm_trace("'%s' execution request from %s as %s user",
1441               pcmk__s(crm_op, operation),
1442               pcmk__s(from_sys, "unknown subsystem"),
1443               pcmk__s(user_name, "current"));
1444 
1445     if (pcmk__str_eq(crm_op, CRM_OP_LRM_FAIL, pcmk__str_none)) {
1446         fail_lrm_resource(input->xml, lrm_state, user_name, from_host,
1447                           from_sys);
1448 
1449     } else if (pcmk__str_eq(crm_op, CRM_OP_REPROBE, pcmk__str_none)
1450                || pcmk__str_eq(operation, CRM_OP_REPROBE, pcmk__str_none)) {
1451         const char *raw_target = NULL;
1452 
1453         if (input->xml != NULL) {
1454             // For CRM_OP_REPROBE, a NULL target means we're targeting all nodes
1455             raw_target = crm_element_value(input->xml, PCMK__META_ON_NODE);
1456         }
1457         handle_reprobe_op(lrm_state, input->msg, from_sys, from_host, user_name,
1458                           is_remote_node, (raw_target == NULL));
1459 
1460     } else if (operation != NULL) {
1461         lrmd_rsc_info_t *rsc = NULL;
1462         xmlNode *xml_rsc = pcmk__xe_first_child(input->xml, PCMK_XE_PRIMITIVE,
1463                                                 NULL, NULL);
1464         gboolean create_rsc = !pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1465                                             pcmk__str_none);
1466         int rc;
1467 
1468         // We can't return anything meaningful without a resource ID
1469         CRM_CHECK((xml_rsc != NULL) && (pcmk__xe_id(xml_rsc) != NULL), return);
1470 
1471         rc = get_lrm_resource(lrm_state, xml_rsc, create_rsc, &rsc);
1472         if (rc == -ENOTCONN) {
1473             synthesize_lrmd_failure(lrm_state, input->xml,
1474                                     PCMK_EXEC_NOT_CONNECTED,
1475                                     PCMK_OCF_UNKNOWN_ERROR,
1476                                     "Not connected to remote executor");
1477             return;
1478 
1479         } else if ((rc < 0) && !create_rsc) {
1480             /* Delete of malformed or nonexistent resource
1481              * (deleting something that does not exist is a success)
1482              */
1483             crm_debug("Not registering resource '%s' for a %s event "
1484                       QB_XS " get-rc=%d (%s) transition-key=%s",
1485                       pcmk__xe_id(xml_rsc), operation,
1486                       rc, pcmk_strerror(rc), pcmk__xe_id(input->xml));
1487             delete_rsc_entry(lrm_state, input, pcmk__xe_id(xml_rsc), NULL,
1488                              pcmk_ok, user_name, true);
1489             return;
1490 
1491         } else if (rc == -EINVAL) {
1492             // Resource operation on malformed resource
1493             crm_err("Invalid resource definition for %s", pcmk__xe_id(xml_rsc));
1494             crm_log_xml_warn(input->msg, "invalid resource");
1495             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1496                                     PCMK_OCF_NOT_CONFIGURED, // fatal error
1497                                     "Invalid resource definition");
1498             return;
1499 
1500         } else if (rc < 0) {
1501             // Error communicating with the executor
1502             crm_err("Could not register resource '%s' with executor: %s "
1503                     QB_XS " rc=%d",
1504                     pcmk__xe_id(xml_rsc), pcmk_strerror(rc), rc);
1505             crm_log_xml_warn(input->msg, "failed registration");
1506             synthesize_lrmd_failure(lrm_state, input->xml, PCMK_EXEC_ERROR,
1507                                     PCMK_OCF_INVALID_PARAM, // hard error
1508                                     "Could not register resource with executor");
1509             return;
1510         }
1511 
1512         if (pcmk__str_eq(operation, PCMK_ACTION_CANCEL, pcmk__str_none)) {
1513             if (!do_lrm_cancel(input, lrm_state, rsc, from_host, from_sys)) {
1514                 crm_log_xml_warn(input->xml, "Bad command");
1515             }
1516 
1517         } else if (pcmk__str_eq(operation, PCMK_ACTION_DELETE,
1518                                 pcmk__str_none)) {
1519             do_lrm_delete(input, lrm_state, rsc, from_sys, from_host,
1520                           crm_rsc_delete, user_name);
1521 
1522         } else {
1523             struct ra_metadata_s *md = NULL;
1524 
1525             /* Getting metadata from cache is OK except for start actions --
1526              * always refresh from the agent for those, in case the resource
1527              * agent was updated.
1528              *
1529              * @TODO Only refresh metadata for starts if the agent actually
1530              * changed (using something like inotify, or a hash or modification
1531              * time of the agent executable).
1532              */
1533             if (strcmp(operation, PCMK_ACTION_START) != 0) {
1534                 md = controld_get_rsc_metadata(lrm_state, rsc,
1535                                                controld_metadata_from_cache);
1536             }
1537 
1538             if ((md == NULL) && crm_op_needs_metadata(rsc->standard,
1539                                                       operation)) {
1540                 /* Most likely, we'll need the agent metadata to record the
1541                  * pending operation and the operation result. Get it now rather
1542                  * than wait until then, so the metadata action doesn't eat into
1543                  * the real action's timeout.
1544                  *
1545                  * @TODO Metadata is retrieved via direct execution of the
1546                  * agent, which has a couple of related issues: the executor
1547                  * should execute agents, not the controller; and metadata for
1548                  * Pacemaker Remote nodes should be collected on those nodes,
1549                  * not locally.
1550                  */
1551                 struct metadata_cb_data *data = NULL;
1552 
1553                 data = new_metadata_cb_data(rsc, input->xml);
1554                 crm_info("Retrieving metadata for %s (%s%s%s:%s) asynchronously",
1555                          rsc->id, rsc->standard,
1556                          ((rsc->provider == NULL)? "" : ":"),
1557                          ((rsc->provider == NULL)? "" : rsc->provider),
1558                          rsc->type);
1559                 (void) lrmd__metadata_async(rsc, metadata_complete,
1560                                             (void *) data);
1561             } else {
1562                 do_lrm_rsc_op(lrm_state, rsc, input->xml, md);
1563             }
1564         }
1565 
1566         lrmd_free_rsc_info(rsc);
1567 
1568     } else {
1569         crm_err("Invalid execution request: unknown command '%s' (bug?)",
1570                 crm_op);
1571         register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
1572     }
1573 }
1574 
1575 static lrmd_event_data_t *
1576 construct_op(const lrm_state_t *lrm_state, const xmlNode *rsc_op,
     /*  */
1577              const char *rsc_id, const char *operation)
1578 {
1579     lrmd_event_data_t *op = NULL;
1580     const char *op_delay = NULL;
1581     const char *op_timeout = NULL;
1582     GHashTable *params = NULL;
1583 
1584     xmlNode *primitive = NULL;
1585     const char *class = NULL;
1586 
1587     const char *transition = NULL;
1588 
1589     pcmk__assert((rsc_id != NULL) && (operation != NULL));
1590 
1591     op = lrmd_new_event(rsc_id, operation, 0);
1592     op->type = lrmd_event_exec_complete;
1593     op->timeout = 0;
1594     op->start_delay = 0;
1595     lrmd__set_result(op, PCMK_OCF_UNKNOWN, PCMK_EXEC_PENDING, NULL);
1596 
1597     if (rsc_op == NULL) {
1598         CRM_LOG_ASSERT(pcmk__str_eq(operation, PCMK_ACTION_STOP,
1599                                     pcmk__str_casei));
1600         op->user_data = NULL;
1601         /* the stop_all_resources() case
1602          * by definition there is no DC (or they'd be shutting
1603          *   us down).
1604          * So we should put our version here.
1605          */
1606         op->params = pcmk__strkey_table(free, free);
1607 
1608         pcmk__insert_dup(op->params, PCMK_XA_CRM_FEATURE_SET, CRM_FEATURE_SET);
1609 
1610         crm_trace("Constructed %s op for %s", operation, rsc_id);
1611         return op;
1612     }
1613 
1614     params = xml2list(rsc_op);
1615     g_hash_table_remove(params, CRM_META "_" PCMK__META_OP_TARGET_RC);
1616 
1617     op_delay = crm_meta_value(params, PCMK_META_START_DELAY);
1618     pcmk__scan_min_int(op_delay, &op->start_delay, 0);
1619 
1620     op_timeout = crm_meta_value(params, PCMK_META_TIMEOUT);
1621     pcmk__scan_min_int(op_timeout, &op->timeout, 0);
1622 
1623     if (pcmk__guint_from_hash(params, CRM_META "_" PCMK_META_INTERVAL, 0,
1624                               &(op->interval_ms)) != pcmk_rc_ok) {
1625         op->interval_ms = 0;
1626     }
1627 
1628     /* Use pcmk_monitor_timeout instead of meta timeout for stonith
1629        recurring monitor, if set */
1630     primitive = pcmk__xe_first_child(rsc_op, PCMK_XE_PRIMITIVE, NULL, NULL);
1631     class = crm_element_value(primitive, PCMK_XA_CLASS);
1632 
1633     if (pcmk_is_set(pcmk_get_ra_caps(class), pcmk_ra_cap_fence_params)
1634             && pcmk__str_eq(operation, PCMK_ACTION_MONITOR, pcmk__str_casei)
1635             && (op->interval_ms > 0)) {
1636 
1637         op_timeout = g_hash_table_lookup(params, "pcmk_monitor_timeout");
1638         if (op_timeout != NULL) {
1639             long long timeout_ms = crm_get_msec(op_timeout);
1640 
1641             op->timeout = (int) QB_MIN(timeout_ms, INT_MAX);
1642         }
1643     }
1644 
1645     if (!pcmk__str_eq(operation, PCMK_ACTION_STOP, pcmk__str_casei)) {
1646         op->params = params;
1647 
1648     } else {
1649         rsc_history_t *entry = NULL;
1650 
1651         if (lrm_state) {
1652             entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
1653         }
1654 
1655         /* If we do not have stop parameters cached, use
1656          * whatever we are given */
1657         if (!entry || !entry->stop_params) {
1658             op->params = params;
1659         } else {
1660             /* Copy the cached parameter list so that we stop the resource
1661              * with the old attributes, not the new ones */
1662             op->params = pcmk__strkey_table(free, free);
1663 
1664             g_hash_table_foreach(params, copy_meta_keys, op->params);
1665             g_hash_table_foreach(entry->stop_params, copy_instance_keys, op->params);
1666             g_hash_table_destroy(params);
1667             params = NULL;
1668         }
1669     }
1670 
1671     /* sanity */
1672     if (op->timeout <= 0) {
1673         op->timeout = op->interval_ms;
1674     }
1675     if (op->start_delay < 0) {
1676         op->start_delay = 0;
1677     }
1678 
1679     transition = crm_element_value(rsc_op, PCMK__XA_TRANSITION_KEY);
1680     CRM_CHECK(transition != NULL, return op);
1681 
1682     op->user_data = pcmk__str_copy(transition);
1683 
1684     if (op->interval_ms != 0) {
1685         if (pcmk__strcase_any_of(operation, PCMK_ACTION_START, PCMK_ACTION_STOP,
1686                                  NULL)) {
1687             crm_err("Start and Stop actions cannot have an interval: %u",
1688                     op->interval_ms);
1689             op->interval_ms = 0;
1690         }
1691     }
1692 
1693     crm_trace("Constructed %s op for %s: interval=%u",
1694               operation, rsc_id, op->interval_ms);
1695 
1696     return op;
1697 }
1698 
1699 /*!
1700  * \internal
1701  * \brief Send a (synthesized) event result
1702  *
1703  * Reply with a synthesized event result directly, as opposed to going through
1704  * the executor.
1705  *
1706  * \param[in]     to_host  Host to send result to
1707  * \param[in]     to_sys   IPC name to send result (NULL for transition engine)
1708  * \param[in]     rsc      Type information about resource the result is for
1709  * \param[in,out] op       Event with result to send
1710  * \param[in]     rsc_id   ID of resource the result is for
1711  */
1712 void
1713 controld_ack_event_directly(const char *to_host, const char *to_sys,
     /*  */
1714                             const lrmd_rsc_info_t *rsc, lrmd_event_data_t *op,
1715                             const char *rsc_id)
1716 {
1717     xmlNode *reply = NULL;
1718     xmlNode *update, *iter;
1719     pcmk__node_status_t *peer = NULL;
1720 
1721     CRM_CHECK(op != NULL, return);
1722     if (op->rsc_id == NULL) {
1723         // op->rsc_id is a (const char *) but lrmd_free_event() frees it
1724         pcmk__assert(rsc_id != NULL);
1725         op->rsc_id = pcmk__str_copy(rsc_id);
1726     }
1727     if (to_sys == NULL) {
1728         to_sys = CRM_SYSTEM_TENGINE;
1729     }
1730 
1731     peer = controld_get_local_node_status();
1732     update = create_node_state_update(peer, node_update_none, NULL,
1733                                       __func__);
1734 
1735     iter = pcmk__xe_create(update, PCMK__XE_LRM);
1736     crm_xml_add(iter, PCMK_XA_ID, controld_globals.our_uuid);
1737     iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCES);
1738     iter = pcmk__xe_create(iter, PCMK__XE_LRM_RESOURCE);
1739 
1740     crm_xml_add(iter, PCMK_XA_ID, op->rsc_id);
1741 
1742     controld_add_resource_history_xml(iter, rsc, op,
1743                                       controld_globals.cluster->priv->node_name);
1744 
1745     /* We don't have the original message ID, so use "direct-ack" (we just need
1746      * something non-NULL for this to create a reply)
1747      *
1748      * @TODO It would be better to use the server, message ID, and task from the
1749      * original request when callers have it available
1750      */
1751     reply = pcmk__new_message(pcmk_ipc_controld, "direct-ack", CRM_SYSTEM_LRMD,
1752                               to_host, to_sys, CRM_OP_INVOKE_LRM, update);
1753 
1754     crm_log_xml_trace(update, "[direct ACK]");
1755 
1756     crm_debug("ACK'ing resource op " PCMK__OP_FMT " from %s: %s",
1757               op->rsc_id, op->op_type, op->interval_ms, op->user_data,
1758               crm_element_value(reply, PCMK_XA_REFERENCE));
1759 
1760     if (relay_message(reply, TRUE) == FALSE) {
1761         crm_log_xml_err(reply, "Unable to route reply");
1762     }
1763 
1764     pcmk__xml_free(update);
1765     pcmk__xml_free(reply);
1766 }
1767 
1768 gboolean
1769 verify_stopped(enum crmd_fsa_state cur_state, int log_level)
     /*  */
1770 {
1771     gboolean res = TRUE;
1772     GList *lrm_state_list = lrm_state_get_list();
1773     GList *state_entry;
1774 
1775     for (state_entry = lrm_state_list; state_entry != NULL; state_entry = state_entry->next) {
1776         lrm_state_t *lrm_state = state_entry->data;
1777 
1778         if (!lrm_state_verify_stopped(lrm_state, cur_state, log_level)) {
1779             /* keep iterating through all even when false is returned */
1780             res = FALSE;
1781         }
1782     }
1783 
1784     controld_set_fsa_input_flags(R_SENT_RSC_STOP);
1785     g_list_free(lrm_state_list); lrm_state_list = NULL;
1786     return res;
1787 }
1788 
1789 struct stop_recurring_action_s {
1790     lrmd_rsc_info_t *rsc;
1791     lrm_state_t *lrm_state;
1792 };
1793 
1794 static gboolean
1795 stop_recurring_action_by_rsc(gpointer key, gpointer value, gpointer user_data)
     /*  */
1796 {
1797     gboolean remove = FALSE;
1798     struct stop_recurring_action_s *event = user_data;
1799     active_op_t *op = value;
1800 
1801     if ((op->interval_ms != 0)
1802         && pcmk__str_eq(op->rsc_id, event->rsc->id, pcmk__str_none)) {
1803 
1804         crm_debug("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id, (char*)key);
1805         remove = !cancel_op(event->lrm_state, event->rsc->id, key, op->call_id, FALSE);
1806     }
1807 
1808     return remove;
1809 }
1810 
1811 static gboolean
1812 stop_recurring_actions(gpointer key, gpointer value, gpointer user_data)
     /*  */
1813 {
1814     gboolean remove = FALSE;
1815     lrm_state_t *lrm_state = user_data;
1816     active_op_t *op = value;
1817 
1818     if (op->interval_ms != 0) {
1819         crm_info("Cancelling op %d for %s (%s)", op->call_id, op->rsc_id,
1820                  (const char *) key);
1821         remove = !cancel_op(lrm_state, op->rsc_id, key, op->call_id, FALSE);
1822     }
1823 
1824     return remove;
1825 }
1826 
1827 /*!
1828  * \internal
1829  * \brief Check whether recurring actions should be cancelled before an action
1830  *
1831  * \param[in] rsc_id       Resource that action is for
1832  * \param[in] action       Action being performed
1833  * \param[in] interval_ms  Operation interval of \p action (in milliseconds)
1834  *
1835  * \return true if recurring actions should be cancelled, otherwise false
1836  */
1837 static bool
1838 should_cancel_recurring(const char *rsc_id, const char *action, guint interval_ms)
     /*  */
1839 {
1840     if (is_remote_lrmd_ra(NULL, NULL, rsc_id) && (interval_ms == 0)
1841         && (strcmp(action, PCMK_ACTION_MIGRATE_TO) == 0)) {
1842         /* Don't stop monitoring a migrating Pacemaker Remote connection
1843          * resource until the entire migration has completed. We must detect if
1844          * the connection is unexpectedly severed, even during a migration.
1845          */
1846         return false;
1847     }
1848 
1849     // Cancel recurring actions before changing resource state
1850     return (interval_ms == 0)
1851             && !pcmk__str_any_of(action, PCMK_ACTION_MONITOR,
1852                                  PCMK_ACTION_NOTIFY, NULL);
1853 }
1854 
1855 /*!
1856  * \internal
1857  * \brief Check whether an action should not be performed at this time
1858  *
1859  * \param[in] operation  Action to be performed
1860  *
1861  * \return Readable description of why action should not be performed,
1862  *         or NULL if it should be performed
1863  */
1864 static const char *
1865 should_nack_action(const char *action)
     /*  */
1866 {
1867     if (pcmk_is_set(controld_globals.fsa_input_register, R_SHUTDOWN)
1868         && pcmk__str_eq(action, PCMK_ACTION_START, pcmk__str_none)) {
1869 
1870         register_fsa_input(C_SHUTDOWN, I_SHUTDOWN, NULL);
1871         return "Not attempting start due to shutdown in progress";
1872     }
1873 
1874     switch (controld_globals.fsa_state) {
1875         case S_NOT_DC:
1876         case S_POLICY_ENGINE:   // Recalculating
1877         case S_TRANSITION_ENGINE:
1878             break;
1879         default:
1880             if (!pcmk__str_eq(action, PCMK_ACTION_STOP, pcmk__str_none)) {
1881                 return "Controller cannot attempt actions at this time";
1882             }
1883             break;
1884     }
1885     return NULL;
1886 }
1887 
1888 static void
1889 do_lrm_rsc_op(lrm_state_t *lrm_state, lrmd_rsc_info_t *rsc, xmlNode *msg,
     /*  */
1890               struct ra_metadata_s *md)
1891 {
1892     int rc;
1893     int call_id = 0;
1894     char *op_id = NULL;
1895     lrmd_event_data_t *op = NULL;
1896     fsa_data_t *msg_data = NULL;
1897     const char *transition = NULL;
1898     const char *operation = NULL;
1899     const char *nack_reason = NULL;
1900 
1901     CRM_CHECK((rsc != NULL) && (msg != NULL), return);
1902 
1903     operation = crm_element_value(msg, PCMK_XA_OPERATION);
1904     CRM_CHECK(!pcmk__str_empty(operation), return);
1905 
1906     transition = crm_element_value(msg, PCMK__XA_TRANSITION_KEY);
1907     if (pcmk__str_empty(transition)) {
1908         crm_log_xml_err(msg, "Missing transition number");
1909     }
1910 
1911     if (lrm_state == NULL) {
1912         // This shouldn't be possible, but provide a failsafe just in case
1913         crm_err("Cannot execute %s of %s: No executor connection "
1914                 QB_XS " transition_key=%s",
1915                 operation, rsc->id, pcmk__s(transition, ""));
1916         synthesize_lrmd_failure(NULL, msg, PCMK_EXEC_INVALID,
1917                                 PCMK_OCF_UNKNOWN_ERROR,
1918                                 "No executor connection");
1919         return;
1920     }
1921 
1922     if (pcmk__str_any_of(operation, PCMK_ACTION_RELOAD,
1923                          PCMK_ACTION_RELOAD_AGENT, NULL)) {
1924         /* Pre-2.1.0 DCs will schedule reload actions only, and 2.1.0+ DCs
1925          * will schedule reload-agent actions only. In either case, we need
1926          * to map that to whatever the resource agent actually supports.
1927          * Default to the OCF 1.1 name.
1928          */
1929         if ((md != NULL)
1930             && pcmk_is_set(md->ra_flags, ra_supports_legacy_reload)) {
1931             operation = PCMK_ACTION_RELOAD;
1932         } else {
1933             operation = PCMK_ACTION_RELOAD_AGENT;
1934         }
1935     }
1936 
1937     op = construct_op(lrm_state, msg, rsc->id, operation);
1938     CRM_CHECK(op != NULL, return);
1939 
1940     if (should_cancel_recurring(rsc->id, operation, op->interval_ms)) {
1941         guint removed = 0;
1942         struct stop_recurring_action_s data;
1943 
1944         data.rsc = rsc;
1945         data.lrm_state = lrm_state;
1946         removed = g_hash_table_foreach_remove(lrm_state->active_ops,
1947                                               stop_recurring_action_by_rsc,
1948                                               &data);
1949 
1950         if (removed) {
1951             crm_debug("Stopped %u recurring operation%s in preparation for "
1952                       PCMK__OP_FMT, removed, pcmk__plural_s(removed),
1953                       rsc->id, operation, op->interval_ms);
1954         }
1955     }
1956 
1957     /* now do the op */
1958     crm_notice("Requesting local execution of %s operation for %s on %s "
1959                QB_XS " transition_key=%s op_key=" PCMK__OP_FMT,
1960                pcmk__readable_action(op->op_type, op->interval_ms), rsc->id,
1961                lrm_state->node_name, pcmk__s(transition, ""), rsc->id,
1962                operation, op->interval_ms);
1963 
1964     nack_reason = should_nack_action(operation);
1965     if (nack_reason != NULL) {
1966         crm_notice("Discarding attempt to perform action %s on %s in state %s "
1967                    "(shutdown=%s)", operation, rsc->id,
1968                    fsa_state2string(controld_globals.fsa_state),
1969                    pcmk__flag_text(controld_globals.fsa_input_register,
1970                                    R_SHUTDOWN));
1971 
1972         lrmd__set_result(op, PCMK_OCF_UNKNOWN_ERROR, PCMK_EXEC_INVALID,
1973                          nack_reason);
1974         controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
1975         lrmd_free_event(op);
1976         free(op_id);
1977         return;
1978     }
1979 
1980     controld_record_pending_op(lrm_state->node_name, rsc, op);
1981 
1982     op_id = pcmk__op_key(rsc->id, op->op_type, op->interval_ms);
1983 
1984     if (op->interval_ms > 0) {
1985         /* cancel it so we can then restart it without conflict */
1986         cancel_op_key(lrm_state, rsc, op_id, FALSE);
1987     }
1988 
1989     rc = controld_execute_resource_agent(lrm_state, rsc->id, op->op_type,
1990                                          op->user_data, op->interval_ms,
1991                                          op->timeout, op->start_delay,
1992                                          op->params, &call_id);
1993     if (rc == pcmk_rc_ok) {
1994         /* record all operations so we can wait
1995          * for them to complete during shutdown
1996          */
1997         char *call_id_s = make_stop_id(rsc->id, call_id);
1998         active_op_t *pending = NULL;
1999 
2000         pending = pcmk__assert_alloc(1, sizeof(active_op_t));
2001         crm_trace("Recording pending op: %d - %s %s", call_id, op_id, call_id_s);
2002 
2003         pending->call_id = call_id;
2004         pending->interval_ms = op->interval_ms;
2005         pending->op_type = pcmk__str_copy(operation);
2006         pending->op_key = pcmk__str_copy(op_id);
2007         pending->rsc_id = pcmk__str_copy(rsc->id);
2008         pending->start_time = time(NULL);
2009         pending->user_data = pcmk__str_copy(op->user_data);
2010         if (crm_element_value_epoch(msg, PCMK_OPT_SHUTDOWN_LOCK,
2011                                     &(pending->lock_time)) != pcmk_ok) {
2012             pending->lock_time = 0;
2013         }
2014         g_hash_table_replace(lrm_state->active_ops, call_id_s, pending);
2015 
2016         if ((op->interval_ms > 0)
2017             && (op->start_delay > START_DELAY_THRESHOLD)) {
2018             int target_rc = PCMK_OCF_OK;
2019 
2020             crm_info("Faking confirmation of %s: execution postponed for over 5 minutes", op_id);
2021             decode_transition_key(op->user_data, NULL, NULL, NULL, &target_rc);
2022             lrmd__set_result(op, target_rc, PCMK_EXEC_DONE, NULL);
2023             controld_ack_event_directly(NULL, NULL, rsc, op, rsc->id);
2024         }
2025 
2026         pending->params = op->params;
2027         op->params = NULL;
2028 
2029     } else if (lrm_state_is_local(lrm_state)) {
2030         crm_err("Could not initiate %s action for resource %s locally: %s "
2031                 QB_XS " rc=%d", operation, rsc->id, pcmk_rc_str(rc), rc);
2032         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2033                        PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2034         process_lrm_event(lrm_state, op, NULL, NULL);
2035         register_fsa_error(C_FSA_INTERNAL, I_FAIL, NULL);
2036 
2037     } else {
2038         crm_err("Could not initiate %s action for resource %s remotely on %s: "
2039                 "%s " QB_XS " rc=%d",
2040                 operation, rsc->id, lrm_state->node_name, pcmk_rc_str(rc), rc);
2041         fake_op_status(lrm_state, op, PCMK_EXEC_NOT_CONNECTED,
2042                        PCMK_OCF_UNKNOWN_ERROR, pcmk_rc_str(rc));
2043         process_lrm_event(lrm_state, op, NULL, NULL);
2044     }
2045 
2046     free(op_id);
2047     lrmd_free_event(op);
2048 }
2049 
2050 void
2051 do_lrm_event(long long action,
     /*  */
2052              enum crmd_fsa_cause cause,
2053              enum crmd_fsa_state cur_state, enum crmd_fsa_input cur_input, fsa_data_t * msg_data)
2054 {
2055     CRM_CHECK(FALSE, return);
2056 }
2057 
2058 static char *
2059 unescape_newlines(const char *string)
     /*  */
2060 {
2061     char *pch = NULL;
2062     char *ret = NULL;
2063     static const char *escaped_newline = "\\n";
2064 
2065     if (!string) {
2066         return NULL;
2067     }
2068 
2069     ret = pcmk__str_copy(string);
2070     pch = strstr(ret, escaped_newline);
2071     while (pch != NULL) {
2072         /* Replace newline escape pattern with actual newline (and a space so we
2073          * don't have to shuffle the rest of the buffer)
2074          */
2075         pch[0] = '\n';
2076         pch[1] = ' ';
2077         pch = strstr(pch, escaped_newline);
2078     }
2079 
2080     return ret;
2081 }
2082 
2083 static bool
2084 did_lrm_rsc_op_fail(lrm_state_t *lrm_state, const char * rsc_id,
     /*  */
2085                     const char * op_type, guint interval_ms)
2086 {
2087     rsc_history_t *entry = NULL;
2088 
2089     CRM_CHECK(lrm_state != NULL, return FALSE);
2090     CRM_CHECK(rsc_id != NULL, return FALSE);
2091     CRM_CHECK(op_type != NULL, return FALSE);
2092 
2093     entry = g_hash_table_lookup(lrm_state->resource_history, rsc_id);
2094     if (entry == NULL || entry->failed == NULL) {
2095         return FALSE;
2096     }
2097 
2098     if (pcmk__str_eq(entry->failed->rsc_id, rsc_id, pcmk__str_none)
2099         && pcmk__str_eq(entry->failed->op_type, op_type, pcmk__str_casei)
2100         && entry->failed->interval_ms == interval_ms) {
2101         return TRUE;
2102     }
2103 
2104     return FALSE;
2105 }
2106 
2107 /*!
2108  * \internal
2109  * \brief Log the result of an executor action (actual or synthesized)
2110  *
2111  * \param[in] op         Executor action to log result for
2112  * \param[in] op_key     Operation key for action
2113  * \param[in] node_name  Name of node action was performed on, if known
2114  * \param[in] confirmed  Whether to log that graph action was confirmed
2115  */
2116 static void
2117 log_executor_event(const lrmd_event_data_t *op, const char *op_key,
     /*  */
2118                    const char *node_name, gboolean confirmed)
2119 {
2120     int log_level = LOG_ERR;
2121     GString *str = g_string_sized_new(100); // reasonable starting size
2122 
2123     pcmk__g_strcat(str,
2124                    "Result of ",
2125                    pcmk__readable_action(op->op_type, op->interval_ms),
2126                    " operation for ", op->rsc_id, NULL);
2127 
2128     if (node_name != NULL) {
2129         pcmk__g_strcat(str, " on ", node_name, NULL);
2130     }
2131 
2132     switch (op->op_status) {
2133         case PCMK_EXEC_DONE:
2134             log_level = LOG_NOTICE;
2135             pcmk__g_strcat(str, ": ", crm_exit_str((crm_exit_t) op->rc), NULL);
2136             break;
2137 
2138         case PCMK_EXEC_TIMEOUT:
2139             pcmk__g_strcat(str,
2140                            ": ", pcmk_exec_status_str(op->op_status), " after ",
2141                            pcmk__readable_interval(op->timeout), NULL);
2142             break;
2143 
2144         case PCMK_EXEC_CANCELLED:
2145             log_level = LOG_INFO;
2146             pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
2147                            NULL);
2148             break;
2149 
2150         default:
2151             pcmk__g_strcat(str, ": ", pcmk_exec_status_str(op->op_status),
2152                            NULL);
2153             break;
2154     }
2155 
2156     if ((op->exit_reason != NULL)
2157         && ((op->op_status != PCMK_EXEC_DONE) || (op->rc != PCMK_OCF_OK))) {
2158 
2159         pcmk__g_strcat(str, " (", op->exit_reason, ")", NULL);
2160     }
2161 
2162     g_string_append(str, " " QB_XS);
2163     g_string_append_printf(str, " graph action %sconfirmed; call=%d key=%s",
2164                            (confirmed? "" : "un"), op->call_id, op_key);
2165     if (op->op_status == PCMK_EXEC_DONE) {
2166         g_string_append_printf(str, " rc=%d", op->rc);
2167     }
2168 
2169     do_crm_log(log_level, "%s", str->str);
2170     g_string_free(str, TRUE);
2171 
2172     /* The services library has already logged the output at info or debug
2173      * level, so just raise to notice if it looks like a failure.
2174      */
2175     if ((op->output != NULL) && (op->rc != PCMK_OCF_OK)) {
2176         char *prefix = crm_strdup_printf(PCMK__OP_FMT "@%s output",
2177                                          op->rsc_id, op->op_type,
2178                                          op->interval_ms, node_name);
2179 
2180         crm_log_output(LOG_NOTICE, prefix, op->output);
2181         free(prefix);
2182     }
2183 }
2184 
2185 void
2186 process_lrm_event(lrm_state_t *lrm_state, lrmd_event_data_t *op,
     /*  */
2187                   active_op_t *pending, const xmlNode *action_xml)
2188 {
2189     char *op_id = NULL;
2190     char *op_key = NULL;
2191 
2192     gboolean remove = FALSE;
2193     gboolean removed = FALSE;
2194     bool need_direct_ack = FALSE;
2195     lrmd_rsc_info_t *rsc = NULL;
2196     const char *node_name = NULL;
2197 
2198     CRM_CHECK(op != NULL, return);
2199     CRM_CHECK(op->rsc_id != NULL, return);
2200 
2201     // Remap new status codes for older DCs
2202     if (compare_version(controld_globals.dc_version, "3.2.0") < 0) {
2203         switch (op->op_status) {
2204             case PCMK_EXEC_NOT_CONNECTED:
2205                 lrmd__set_result(op, PCMK_OCF_CONNECTION_DIED,
2206                                  PCMK_EXEC_ERROR, op->exit_reason);
2207                 break;
2208             case PCMK_EXEC_INVALID:
2209                 lrmd__set_result(op, CRM_DIRECT_NACK_RC, PCMK_EXEC_ERROR,
2210                                  op->exit_reason);
2211                 break;
2212             default:
2213                 break;
2214         }
2215     }
2216 
2217     op_id = make_stop_id(op->rsc_id, op->call_id);
2218     op_key = pcmk__op_key(op->rsc_id, op->op_type, op->interval_ms);
2219 
2220     // Get resource info if available (from executor state or action XML)
2221     if (lrm_state) {
2222         rsc = lrm_state_get_rsc_info(lrm_state, op->rsc_id, 0);
2223     }
2224     if ((rsc == NULL) && action_xml) {
2225         xmlNode *xml = pcmk__xe_first_child(action_xml, PCMK_XE_PRIMITIVE, NULL,
2226                                             NULL);
2227 
2228         const char *standard = crm_element_value(xml, PCMK_XA_CLASS);
2229         const char *provider = crm_element_value(xml, PCMK_XA_PROVIDER);
2230         const char *type = crm_element_value(xml, PCMK_XA_TYPE);
2231 
2232         if (standard && type) {
2233             crm_info("%s agent information not cached, using %s%s%s:%s from action XML",
2234                      op->rsc_id, standard,
2235                      (provider? ":" : ""), (provider? provider : ""), type);
2236             rsc = lrmd_new_rsc_info(op->rsc_id, standard, provider, type);
2237         } else {
2238             crm_err("Can't process %s result because %s agent information not cached or in XML",
2239                     op_key, op->rsc_id);
2240         }
2241     }
2242 
2243     // Get node name if available (from executor state or action XML)
2244     if (lrm_state) {
2245         node_name = lrm_state->node_name;
2246     } else if (action_xml) {
2247         node_name = crm_element_value(action_xml, PCMK__META_ON_NODE);
2248     }
2249 
2250     if(pending == NULL) {
2251         remove = TRUE;
2252         if (lrm_state) {
2253             pending = g_hash_table_lookup(lrm_state->active_ops, op_id);
2254         }
2255     }
2256 
2257     if (op->op_status == PCMK_EXEC_ERROR) {
2258         switch(op->rc) {
2259             case PCMK_OCF_NOT_RUNNING:
2260             case PCMK_OCF_RUNNING_PROMOTED:
2261             case PCMK_OCF_DEGRADED:
2262             case PCMK_OCF_DEGRADED_PROMOTED:
2263                 // Leave it to the TE/scheduler to decide if this is an error
2264                 op->op_status = PCMK_EXEC_DONE;
2265                 break;
2266             default:
2267                 /* Nothing to do */
2268                 break;
2269         }
2270     }
2271 
2272     if (op->op_status != PCMK_EXEC_CANCELLED) {
2273         /* We might not record the result, so directly acknowledge it to the
2274          * originator instead, so it doesn't time out waiting for the result
2275          * (especially important if part of a transition).
2276          */
2277         need_direct_ack = TRUE;
2278 
2279         if (controld_action_is_recordable(op->op_type)) {
2280             if (node_name && rsc) {
2281                 // We should record the result, and happily, we can
2282                 time_t lock_time = (pending == NULL)? 0 : pending->lock_time;
2283 
2284                 controld_update_resource_history(node_name, rsc, op, lock_time);
2285                 need_direct_ack = FALSE;
2286 
2287             } else if (op->rsc_deleted) {
2288                 /* We shouldn't record the result (likely the resource was
2289                  * refreshed, cleaned, or removed while this operation was
2290                  * in flight).
2291                  */
2292                 crm_notice("Not recording %s result in CIB because "
2293                            "resource information was removed since it was initiated",
2294                            op_key);
2295             } else {
2296                 /* This shouldn't be possible; the executor didn't consider the
2297                  * resource deleted, but we couldn't find resource or node
2298                  * information.
2299                  */
2300                 crm_err("Unable to record %s result in CIB: %s", op_key,
2301                         (node_name? "No resource information" : "No node name"));
2302             }
2303         }
2304 
2305     } else if (op->interval_ms == 0) {
2306         /* A non-recurring operation was cancelled. Most likely, the
2307          * never-initiated action was removed from the executor's pending
2308          * operations list upon resource removal.
2309          */
2310         need_direct_ack = TRUE;
2311 
2312     } else if (pending == NULL) {
2313         /* This recurring operation was cancelled, but was not pending. No
2314          * transition actions are waiting on it, nothing needs to be done.
2315          */
2316 
2317     } else if (op->user_data == NULL) {
2318         /* This recurring operation was cancelled and pending, but we don't
2319          * have a transition key. This should never happen.
2320          */
2321         crm_err("Recurring operation %s was cancelled without transition information",
2322                 op_key);
2323 
2324     } else if (pcmk_is_set(pending->flags, active_op_remove)) {
2325         /* This recurring operation was cancelled (by us) and pending, and we
2326          * have been waiting for it to finish.
2327          */
2328         if (lrm_state) {
2329             controld_delete_action_history(op);
2330         }
2331 
2332         /* Directly acknowledge failed recurring actions here. The above call to
2333          * controld_delete_action_history() will not erase any corresponding
2334          * last_failure entry, which means that the DC won't confirm the
2335          * cancellation via process_op_deletion(), and the transition would
2336          * otherwise wait for the action timer to pop.
2337          */
2338         if (did_lrm_rsc_op_fail(lrm_state, pending->rsc_id,
2339                                 pending->op_type, pending->interval_ms)) {
2340             need_direct_ack = TRUE;
2341         }
2342 
2343     } else if (op->rsc_deleted) {
2344         /* This recurring operation was cancelled (but not by us, and the
2345          * executor does not have resource information, likely due to resource
2346          * cleanup, refresh, or removal) and pending.
2347          */
2348         crm_debug("Recurring op %s was cancelled due to resource deletion",
2349                   op_key);
2350         need_direct_ack = TRUE;
2351 
2352     } else {
2353         /* This recurring operation was cancelled (but not by us, likely by the
2354          * executor before stopping the resource) and pending. We don't need to
2355          * do anything special.
2356          */
2357     }
2358 
2359     if (need_direct_ack) {
2360         controld_ack_event_directly(NULL, NULL, NULL, op, op->rsc_id);
2361     }
2362 
2363     if(remove == FALSE) {
2364         /* The caller will do this afterwards, but keep the logging consistent */
2365         removed = TRUE;
2366 
2367     } else if (lrm_state && ((op->interval_ms == 0)
2368                              || (op->op_status == PCMK_EXEC_CANCELLED))) {
2369 
2370         gboolean found = g_hash_table_remove(lrm_state->active_ops, op_id);
2371 
2372         if (op->interval_ms != 0) {
2373             removed = TRUE;
2374         } else if (found) {
2375             removed = TRUE;
2376             crm_trace("Op %s (call=%d, stop-id=%s, remaining=%u): Confirmed",
2377                       op_key, op->call_id, op_id,
2378                       g_hash_table_size(lrm_state->active_ops));
2379         }
2380     }
2381 
2382     log_executor_event(op, op_key, node_name, removed);
2383 
2384     if (lrm_state) {
2385         if (!pcmk__str_eq(op->op_type, PCMK_ACTION_META_DATA,
2386                           pcmk__str_casei)) {
2387             crmd_alert_resource_op(lrm_state->node_name, op);
2388         } else if (rsc && (op->rc == PCMK_OCF_OK)) {
2389             char *metadata = unescape_newlines(op->output);
2390 
2391             controld_cache_metadata(lrm_state->metadata_cache, rsc, metadata);
2392             free(metadata);
2393         }
2394     }
2395 
2396     if (op->rsc_deleted) {
2397         crm_info("Deletion of resource '%s' complete after %s", op->rsc_id, op_key);
2398         if (lrm_state) {
2399             delete_rsc_entry(lrm_state, NULL, op->rsc_id, NULL, pcmk_ok, NULL,
2400                              true);
2401         }
2402     }
2403 
2404     /* If a shutdown was escalated while operations were pending,
2405      * then the FSA will be stalled right now... allow it to continue
2406      */
2407     controld_trigger_fsa();
2408     if (lrm_state && rsc) {
2409         update_history_cache(lrm_state, rsc, op);
2410     }
2411 
2412     lrmd_free_rsc_info(rsc);
2413     free(op_key);
2414     free(op_id);
2415 }
/* */
root/daemons/controld/controld_execd.c

DEFINITIONS
