root/lrmd/lrmd.c

/* */

DEFINITIONS

1. log_finished
2. log_execute
3. normalize_action_name
4. build_rsc_from_xml
5. create_lrmd_cmd
6. free_lrmd_cmd
7. stonith_recurring_op_helper
8. start_delay_helper
9. merge_recurring_duplicate
10. schedule_lrmd_cmd
11. send_reply
12. send_client_notify
13. time_diff_ms
14. cmd_original_times
15. send_cmd_complete_notify
16. send_generic_notify
17. cmd_reset
18. cmd_finalize
19. pattern_matched
20. hb2uniform_rc
21. ocf2uniform_rc
22. stonith2uniform_rc
23. nagios2uniform_rc
24. get_uniform_rc
25. action_get_uniform_rc
26. notify_of_new_client
27. parse_exit_reason
28. client_disconnect_cleanup
29. action_complete
30. stonith_action_complete
31. lrmd_stonith_callback
32. stonith_connection_failed
33. lrmd_rsc_execute_stonith
34. lrmd_rsc_execute_service_lib
35. lrmd_rsc_execute
36. lrmd_rsc_dispatch
37. free_rsc
38. process_lrmd_signon
39. process_lrmd_rsc_register
40. process_lrmd_get_rsc_info
41. process_lrmd_rsc_unregister
42. process_lrmd_rsc_exec
43. cancel_op
44. cancel_all_recurring
45. process_lrmd_rsc_cancel
46. process_lrmd_message

   1 /*
   2  * Copyright (c) 2012 David Vossel <davidvossel@gmail.com>
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  *
  18  */
  19 
  20 #include <crm_internal.h>
  21 
  22 #include <glib.h>
  23 #include <unistd.h>
  24 
  25 #include <crm/crm.h>
  26 #include <crm/services.h>
  27 #include <crm/common/mainloop.h>
  28 #include <crm/common/ipc.h>
  29 #include <crm/common/ipcs.h>
  30 #include <crm/msg_xml.h>
  31 
  32 #include <lrmd_private.h>
  33 
  34 #ifdef HAVE_SYS_TIMEB_H
  35 #  include <sys/timeb.h>
  36 #endif
  37 
  38 #define EXIT_REASON_MAX_LEN 128
  39 
  40 GHashTable *rsc_list = NULL;
  41 
  42 typedef struct lrmd_cmd_s {
  43     int timeout;
  44     int interval;
  45     int start_delay;
  46     int timeout_orig;
  47 
  48     int call_id;
  49     int exec_rc;
  50     int lrmd_op_status;
  51 
  52     int call_opts;
  53     /* Timer ids, must be removed on cmd destruction. */
  54     int delay_id;
  55     int stonith_recurring_id;
  56 
  57     int rsc_deleted;
  58 
  59     int service_flags;
  60 
  61     char *client_id;
  62     char *origin;
  63     char *rsc_id;
  64     char *action;
  65     char *real_action;
  66     char *exit_reason;
  67     char *output;
  68     char *userdata_str;
  69 
  70     /* when set, this cmd should go through a container wrapper */
  71     const char *isolation_wrapper;
  72 
  73 #ifdef HAVE_SYS_TIMEB_H
  74     /* recurring and systemd operations may involve more than one lrmd command
  75      * per operation, so they need info about original and most recent
  76      */
  77     struct timeb t_first_run;   /* Timestamp of when op first ran */
  78     struct timeb t_run;         /* Timestamp of when op most recently ran */
  79     struct timeb t_first_queue; /* Timestamp of when op first was queued */
  80     struct timeb t_queue;       /* Timestamp of when op most recently was queued */
  81     struct timeb t_rcchange;    /* Timestamp of last rc change */
  82 #endif
  83 
  84     int first_notify_sent;
  85     int last_notify_rc;
  86     int last_notify_op_status;
  87     int last_pid;
  88 
  89     GHashTable *params;
  90 } lrmd_cmd_t;
  91 
  92 static void cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc);
  93 static gboolean lrmd_rsc_dispatch(gpointer user_data);
  94 static void cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id);
  95 
  96 static void
  97 log_finished(lrmd_cmd_t * cmd, int exec_time, int queue_time)
     /*  */
  98 {
  99     char pid_str[32] = { 0, };
 100     int log_level = LOG_INFO;
 101 
 102     if (cmd->last_pid) {
 103         snprintf(pid_str, 32, "%d", cmd->last_pid);
 104     }
 105 
 106     if (safe_str_eq(cmd->action, "monitor")) {
 107         log_level = LOG_DEBUG;
 108     }
 109 #ifdef HAVE_SYS_TIMEB_H
 110     do_crm_log(log_level,
 111                "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d exec-time:%dms queue-time:%dms",
 112                cmd->rsc_id, cmd->action, cmd->call_id, cmd->last_pid ? "pid:" : "", pid_str,
 113                cmd->exec_rc, exec_time, queue_time);
 114 #else
 115     do_crm_log(log_level, "finished - rsc:%s action:%s call_id:%d %s%s exit-code:%d",
 116                cmd->rsc_id,
 117                cmd->action, cmd->call_id, cmd->last_pid ? "pid:" : "", pid_str, cmd->exec_rc);
 118 #endif
 119 }
 120 
 121 static void
 122 log_execute(lrmd_cmd_t * cmd)
     /*  */
 123 {
 124     int log_level = LOG_INFO;
 125 
 126     if (safe_str_eq(cmd->action, "monitor")) {
 127         log_level = LOG_DEBUG;
 128     }
 129 
 130     do_crm_log(log_level, "executing - rsc:%s action:%s call_id:%d",
 131                cmd->rsc_id, cmd->action, cmd->call_id);
 132 }
 133 
 134 static const char *
 135 normalize_action_name(lrmd_rsc_t * rsc, const char *action)
     /*  */
 136 {
 137     if (safe_str_eq(action, "monitor") &&
 138         (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_LSB) ||
 139          safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_SERVICE)
 140          || safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_SYSTEMD))) {
 141         return "status";
 142     }
 143     return action;
 144 }
 145 
 146 static lrmd_rsc_t *
 147 build_rsc_from_xml(xmlNode * msg)
     /*  */
 148 {
 149     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
 150     lrmd_rsc_t *rsc = NULL;
 151 
 152     rsc = calloc(1, sizeof(lrmd_rsc_t));
 153 
 154     crm_element_value_int(msg, F_LRMD_CALLOPTS, &rsc->call_opts);
 155 
 156     rsc->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
 157     rsc->class = crm_element_value_copy(rsc_xml, F_LRMD_CLASS);
 158     rsc->provider = crm_element_value_copy(rsc_xml, F_LRMD_PROVIDER);
 159     rsc->type = crm_element_value_copy(rsc_xml, F_LRMD_TYPE);
 160     rsc->work = mainloop_add_trigger(G_PRIORITY_HIGH, lrmd_rsc_dispatch, rsc);
 161     return rsc;
 162 }
 163 
 164 static lrmd_cmd_t *
 165 create_lrmd_cmd(xmlNode * msg, crm_client_t * client, lrmd_rsc_t *rsc)
     /*  */
 166 {
 167     int call_options = 0;
 168     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, msg, LOG_ERR);
 169     lrmd_cmd_t *cmd = NULL;
 170 
 171     cmd = calloc(1, sizeof(lrmd_cmd_t));
 172 
 173     crm_element_value_int(msg, F_LRMD_CALLOPTS, &call_options);
 174     cmd->call_opts = call_options;
 175     cmd->client_id = strdup(client->id);
 176 
 177     crm_element_value_int(msg, F_LRMD_CALLID, &cmd->call_id);
 178     crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &cmd->interval);
 179     crm_element_value_int(rsc_xml, F_LRMD_TIMEOUT, &cmd->timeout);
 180     crm_element_value_int(rsc_xml, F_LRMD_RSC_START_DELAY, &cmd->start_delay);
 181     cmd->timeout_orig = cmd->timeout;
 182 
 183     cmd->origin = crm_element_value_copy(rsc_xml, F_LRMD_ORIGIN);
 184     cmd->action = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ACTION);
 185     cmd->userdata_str = crm_element_value_copy(rsc_xml, F_LRMD_RSC_USERDATA_STR);
 186     cmd->rsc_id = crm_element_value_copy(rsc_xml, F_LRMD_RSC_ID);
 187 
 188     cmd->params = xml2list(rsc_xml);
 189     cmd->isolation_wrapper = g_hash_table_lookup(cmd->params, "CRM_meta_isolation_wrapper");
 190 
 191     if (cmd->isolation_wrapper) {
 192         if (g_hash_table_lookup(cmd->params, "CRM_meta_isolation_instance") == NULL) {
 193             g_hash_table_insert(cmd->params, strdup("CRM_meta_isolation_instance"), strdup(rsc->rsc_id));
 194         }
 195         if (rsc->provider) {
 196             g_hash_table_insert(cmd->params, strdup("CRM_meta_provider"), strdup(rsc->provider));
 197         }
 198         g_hash_table_insert(cmd->params, strdup("CRM_meta_class"), strdup(rsc->class));
 199         g_hash_table_insert(cmd->params, strdup("CRM_meta_type"), strdup(rsc->type));
 200     }
 201 
 202     if (safe_str_eq(g_hash_table_lookup(cmd->params, "CRM_meta_on_fail"), "block")) {
 203         crm_debug("Setting flag to leave pid group on timeout and only kill action pid for %s_%s_%d", cmd->rsc_id, cmd->action, cmd->interval);
 204         cmd->service_flags |= SVC_ACTION_LEAVE_GROUP;
 205     }
 206     return cmd;
 207 }
 208 
 209 static void
 210 free_lrmd_cmd(lrmd_cmd_t * cmd)
     /*  */
 211 {
 212     if (cmd->stonith_recurring_id) {
 213         g_source_remove(cmd->stonith_recurring_id);
 214     }
 215     if (cmd->delay_id) {
 216         g_source_remove(cmd->delay_id);
 217     }
 218     if (cmd->params) {
 219         g_hash_table_destroy(cmd->params);
 220     }
 221     free(cmd->origin);
 222     free(cmd->action);
 223     free(cmd->real_action);
 224     free(cmd->userdata_str);
 225     free(cmd->rsc_id);
 226     free(cmd->output);
 227     free(cmd->exit_reason);
 228     free(cmd->client_id);
 229     free(cmd);
 230 }
 231 
 232 static gboolean
 233 stonith_recurring_op_helper(gpointer data)
     /*  */
 234 {
 235     lrmd_cmd_t *cmd = data;
 236     lrmd_rsc_t *rsc;
 237 
 238     cmd->stonith_recurring_id = 0;
 239 
 240     if (!cmd->rsc_id) {
 241         return FALSE;
 242     }
 243 
 244     rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
 245 
 246     CRM_ASSERT(rsc != NULL);
 247     /* take it out of recurring_ops list, and put it in the pending ops
 248      * to be executed */
 249     rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
 250     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 251 #ifdef HAVE_SYS_TIMEB_H
 252     ftime(&cmd->t_queue);
 253     if (cmd->t_first_queue.time == 0) {
 254         cmd->t_first_queue = cmd->t_queue;
 255     }
 256 #endif
 257     mainloop_set_trigger(rsc->work);
 258 
 259     return FALSE;
 260 }
 261 
 262 static gboolean
 263 start_delay_helper(gpointer data)
     /*  */
 264 {
 265     lrmd_cmd_t *cmd = data;
 266     lrmd_rsc_t *rsc = NULL;
 267 
 268     cmd->delay_id = 0;
 269     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 270 
 271     if (rsc) {
 272         mainloop_set_trigger(rsc->work);
 273     }
 274 
 275     return FALSE;
 276 }
 277 
 278 static gboolean
 279 merge_recurring_duplicate(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
     /*  */
 280 {
 281     GListPtr gIter = NULL;
 282     lrmd_cmd_t * dup = NULL;
 283     gboolean dup_pending = FALSE;
 284 
 285     if (cmd->interval == 0) {
 286         return 0;
 287     }
 288 
 289     for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
 290         dup = gIter->data;
 291         if (safe_str_eq(cmd->action, dup->action) && cmd->interval == dup->interval) {
 292             dup_pending = TRUE;
 293             goto merge_dup;
 294         }
 295     }
 296 
 297     /* if dup is in recurring_ops list, that means it has already executed
 298      * and is in the interval loop. we can't just remove it in this case. */
 299     for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
 300         dup = gIter->data;
 301         if (safe_str_eq(cmd->action, dup->action) && cmd->interval == dup->interval) {
 302             goto merge_dup;
 303         }
 304     }
 305 
 306     return FALSE;
 307 merge_dup:
 308 
 309 
 310     /* This should not occur, if it does we need to investigate in the crmd
 311      * how something like this is possible */
 312     crm_warn("Duplicate recurring op entry detected (%s_%s_%d), merging with previous op entry",
 313             rsc->rsc_id,
 314             normalize_action_name(rsc, dup->action),
 315             dup->interval);
 316 
 317     /* merge */
 318     dup->first_notify_sent = 0;
 319     free(dup->userdata_str);
 320     dup->userdata_str = cmd->userdata_str;
 321     cmd->userdata_str = NULL;
 322     dup->call_id = cmd->call_id;
 323 
 324     if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) {
 325         /* if we are waiting for the next interval, kick it off now */
 326         if (dup_pending == TRUE) {
 327             g_source_remove(cmd->stonith_recurring_id);
 328             cmd->stonith_recurring_id = 0;
 329             stonith_recurring_op_helper(cmd);
 330         }
 331 
 332     } else if (dup_pending == FALSE) {
 333         /* if we've already handed this to the service lib, kick off an early execution */
 334         services_action_kick(rsc->rsc_id, normalize_action_name(rsc, dup->action), dup->interval);
 335     }
 336     free_lrmd_cmd(cmd);
 337 
 338     return TRUE;
 339 }
 340 
 341 static void
 342 schedule_lrmd_cmd(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
     /*  */
 343 {
 344     gboolean dup_processed = FALSE;
 345     CRM_CHECK(cmd != NULL, return);
 346     CRM_CHECK(rsc != NULL, return);
 347 
 348     crm_trace("Scheduling %s on %s", cmd->action, rsc->rsc_id);
 349 
 350     dup_processed = merge_recurring_duplicate(rsc, cmd);
 351     if (dup_processed) {
 352         /* duplicate recurring cmd found, cmds merged */
 353         return;
 354     }
 355 
 356     /* crmd expects lrmd to automatically cancel recurring ops before rsc stops. */
 357     if (rsc && safe_str_eq(cmd->action, "stop")) {
 358         cancel_all_recurring(rsc, NULL);
 359     }
 360 
 361     rsc->pending_ops = g_list_append(rsc->pending_ops, cmd);
 362 #ifdef HAVE_SYS_TIMEB_H
 363     ftime(&cmd->t_queue);
 364     if (cmd->t_first_queue.time == 0) {
 365         cmd->t_first_queue = cmd->t_queue;
 366     }
 367 #endif
 368     mainloop_set_trigger(rsc->work);
 369 
 370     if (cmd->start_delay) {
 371         cmd->delay_id = g_timeout_add(cmd->start_delay, start_delay_helper, cmd);
 372     }
 373 }
 374 
 375 static void
 376 send_reply(crm_client_t * client, int rc, uint32_t id, int call_id)
     /*  */
 377 {
 378     int send_rc = 0;
 379     xmlNode *reply = NULL;
 380 
 381     reply = create_xml_node(NULL, T_LRMD_REPLY);
 382     crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
 383     crm_xml_add_int(reply, F_LRMD_RC, rc);
 384     crm_xml_add_int(reply, F_LRMD_CALLID, call_id);
 385 
 386     send_rc = lrmd_server_send_reply(client, id, reply);
 387 
 388     free_xml(reply);
 389     if (send_rc < 0) {
 390         crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
 391     }
 392 }
 393 
 394 static void
 395 send_client_notify(gpointer key, gpointer value, gpointer user_data)
     /*  */
 396 {
 397     xmlNode *update_msg = user_data;
 398     crm_client_t *client = value;
 399     int rc;
 400 
 401     if (client == NULL) {
 402         crm_err("Asked to send event to  NULL client");
 403         return;
 404     } else if (client->name == NULL) {
 405         crm_trace("Asked to send event to client with no name");
 406         return;
 407     }
 408 
 409     rc = lrmd_server_send_notify(client, update_msg);
 410     if ((rc <= 0) && (rc != -ENOTCONN)) {
 411         crm_warn("Could not notify client %s/%s: %s " CRM_XS " rc=%d",
 412                  client->name, client->id,
 413                  (rc? pcmk_strerror(rc) : "no data sent"), rc);
 414     }
 415 }
 416 
 417 #ifdef HAVE_SYS_TIMEB_H
 418 /*!
 419  * \internal
 420  * \brief Return difference between two times in milliseconds
 421  *
 422  * \param[in] now  More recent time (or NULL to use current time)
 423  * \param[in] old  Earlier time
 424  *
 425  * \return milliseconds difference (or 0 if old is NULL or has time zero)
 426  */
 427 static int
 428 time_diff_ms(struct timeb *now, struct timeb *old)
     /*  */
 429 {
 430     struct timeb local_now = { 0, };
 431 
 432     if (now == NULL) {
 433         ftime(&local_now);
 434         now = &local_now;
 435     }
 436     if ((old == NULL) || (old->time == 0)) {
 437         return 0;
 438     }
 439     return difftime(now->time, old->time) * 1000 + now->millitm - old->millitm;
 440 }
 441 
 442 /*!
 443  * \internal
 444  * \brief Reset a command's operation times to their original values.
 445  *
 446  * Reset a command's run and queued timestamps to the timestamps of the original
 447  * command, so we report the entire time since then and not just the time since
 448  * the most recent command (for recurring and systemd operations).
 449  *
 450  * /param[in] cmd  LRMD command object to reset
 451  *
 452  * /note It's not obvious what the queued time should be for a systemd
 453  * start/stop operation, which might go like this:
 454  *   initial command queued 5ms, runs 3s
 455  *   monitor command queued 10ms, runs 10s
 456  *   monitor command queued 10ms, runs 10s
 457  * Is the queued time for that operation 5ms, 10ms or 25ms? The current
 458  * implementation will report 5ms. If it's 25ms, then we need to
 459  * subtract 20ms from the total exec time so as not to count it twice.
 460  * We can implement that later if it matters to anyone ...
 461  */
 462 static void
 463 cmd_original_times(lrmd_cmd_t * cmd)
     /*  */
 464 {
 465     cmd->t_run = cmd->t_first_run;
 466     cmd->t_queue = cmd->t_first_queue;
 467 }
 468 #endif
 469 
 470 static void
 471 send_cmd_complete_notify(lrmd_cmd_t * cmd)
     /*  */
 472 {
 473     int exec_time = 0;
 474     int queue_time = 0;
 475     xmlNode *notify = NULL;
 476 
 477 #ifdef HAVE_SYS_TIMEB_H
 478     exec_time = time_diff_ms(NULL, &cmd->t_run);
 479     queue_time = time_diff_ms(&cmd->t_run, &cmd->t_queue);
 480 #endif
 481 
 482     log_finished(cmd, exec_time, queue_time);
 483 
 484     /* if the first notify result for a cmd has already been sent earlier, and the
 485      * the option to only send notifies on result changes is set. Check to see
 486      * if the last result is the same as the new one. If so, suppress this update */
 487     if (cmd->first_notify_sent && (cmd->call_opts & lrmd_opt_notify_changes_only)) {
 488         if (cmd->last_notify_rc == cmd->exec_rc &&
 489             cmd->last_notify_op_status == cmd->lrmd_op_status) {
 490 
 491             /* only send changes */
 492             return;
 493         }
 494 
 495     }
 496 
 497     cmd->first_notify_sent = 1;
 498     cmd->last_notify_rc = cmd->exec_rc;
 499     cmd->last_notify_op_status = cmd->lrmd_op_status;
 500 
 501     notify = create_xml_node(NULL, T_LRMD_NOTIFY);
 502 
 503     crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
 504     crm_xml_add_int(notify, F_LRMD_TIMEOUT, cmd->timeout);
 505     crm_xml_add_int(notify, F_LRMD_RSC_INTERVAL, cmd->interval);
 506     crm_xml_add_int(notify, F_LRMD_RSC_START_DELAY, cmd->start_delay);
 507     crm_xml_add_int(notify, F_LRMD_EXEC_RC, cmd->exec_rc);
 508     crm_xml_add_int(notify, F_LRMD_OP_STATUS, cmd->lrmd_op_status);
 509     crm_xml_add_int(notify, F_LRMD_CALLID, cmd->call_id);
 510     crm_xml_add_int(notify, F_LRMD_RSC_DELETED, cmd->rsc_deleted);
 511 
 512 #ifdef HAVE_SYS_TIMEB_H
 513     crm_xml_add_int(notify, F_LRMD_RSC_RUN_TIME, cmd->t_run.time);
 514     crm_xml_add_int(notify, F_LRMD_RSC_RCCHANGE_TIME, cmd->t_rcchange.time);
 515     crm_xml_add_int(notify, F_LRMD_RSC_EXEC_TIME, exec_time);
 516     crm_xml_add_int(notify, F_LRMD_RSC_QUEUE_TIME, queue_time);
 517 #endif
 518 
 519     crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_RSC_EXEC);
 520     crm_xml_add(notify, F_LRMD_RSC_ID, cmd->rsc_id);
 521     if(cmd->real_action) {
 522         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->real_action);
 523     } else {
 524         crm_xml_add(notify, F_LRMD_RSC_ACTION, cmd->action);
 525     }
 526     crm_xml_add(notify, F_LRMD_RSC_USERDATA_STR, cmd->userdata_str);
 527     crm_xml_add(notify, F_LRMD_RSC_OUTPUT, cmd->output);
 528     crm_xml_add(notify, F_LRMD_RSC_EXIT_REASON, cmd->exit_reason);
 529 
 530     if (cmd->params) {
 531         char *key = NULL;
 532         char *value = NULL;
 533         GHashTableIter iter;
 534 
 535         xmlNode *args = create_xml_node(notify, XML_TAG_ATTRS);
 536 
 537         g_hash_table_iter_init(&iter, cmd->params);
 538         while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
 539             hash2smartfield((gpointer) key, (gpointer) value, args);
 540         }
 541     }
 542     if (cmd->client_id && (cmd->call_opts & lrmd_opt_notify_orig_only)) {
 543         crm_client_t *client = crm_client_get_by_id(cmd->client_id);
 544 
 545         if (client) {
 546             send_client_notify(client->id, client, notify);
 547         }
 548     } else if (client_connections != NULL) {
 549         g_hash_table_foreach(client_connections, send_client_notify, notify);
 550     }
 551 
 552     free_xml(notify);
 553 }
 554 
 555 static void
 556 send_generic_notify(int rc, xmlNode * request)
     /*  */
 557 {
 558     if (client_connections != NULL) {
 559         int call_id = 0;
 560         xmlNode *notify = NULL;
 561         xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
 562         const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
 563         const char *op = crm_element_value(request, F_LRMD_OPERATION);
 564 
 565         crm_element_value_int(request, F_LRMD_CALLID, &call_id);
 566 
 567         notify = create_xml_node(NULL, T_LRMD_NOTIFY);
 568         crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
 569         crm_xml_add_int(notify, F_LRMD_RC, rc);
 570         crm_xml_add_int(notify, F_LRMD_CALLID, call_id);
 571         crm_xml_add(notify, F_LRMD_OPERATION, op);
 572         crm_xml_add(notify, F_LRMD_RSC_ID, rsc_id);
 573 
 574         g_hash_table_foreach(client_connections, send_client_notify, notify);
 575 
 576         free_xml(notify);
 577     }
 578 }
 579 
 580 static void
 581 cmd_reset(lrmd_cmd_t * cmd)
     /*  */
 582 {
 583     cmd->lrmd_op_status = 0;
 584     cmd->last_pid = 0;
 585     memset(&cmd->t_run, 0, sizeof(cmd->t_run));
 586     memset(&cmd->t_queue, 0, sizeof(cmd->t_queue));
 587     free(cmd->exit_reason);
 588     cmd->exit_reason = NULL;
 589     free(cmd->output);
 590     cmd->output = NULL;
 591 
 592 }
 593 
 594 static void
 595 cmd_finalize(lrmd_cmd_t * cmd, lrmd_rsc_t * rsc)
     /*  */
 596 {
 597     crm_trace("Resource operation rsc:%s action:%s completed (%p %p)", cmd->rsc_id, cmd->action,
 598               rsc ? rsc->active : NULL, cmd);
 599 
 600     if (rsc && (rsc->active == cmd)) {
 601         rsc->active = NULL;
 602         mainloop_set_trigger(rsc->work);
 603     }
 604 
 605     if (!rsc) {
 606         cmd->rsc_deleted = 1;
 607     }
 608 
 609     /* reset original timeout so client notification has correct information */
 610     cmd->timeout = cmd->timeout_orig;
 611 
 612     send_cmd_complete_notify(cmd);
 613 
 614     if (cmd->interval && (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED)) {
 615         if (rsc) {
 616             rsc->recurring_ops = g_list_remove(rsc->recurring_ops, cmd);
 617             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
 618         }
 619         free_lrmd_cmd(cmd);
 620     } else if (cmd->interval == 0) {
 621         if (rsc) {
 622             rsc->pending_ops = g_list_remove(rsc->pending_ops, cmd);
 623         }
 624         free_lrmd_cmd(cmd);
 625     } else {
 626         /* Clear all the values pertaining just to the last iteration of a recurring op. */
 627         cmd_reset(cmd);
 628     }
 629 }
 630 
 631 #if SUPPORT_HEARTBEAT
 632 static int pattern_matched(const char *pat, const char *str)
     /*  */
 633 {
 634     if (g_pattern_match_simple(pat, str)) {
 635         crm_debug("RA output matched stopped pattern [%s]", pat);
 636         return TRUE;
 637     }
 638     return FALSE;
 639 }
 640 
 641 static int
 642 hb2uniform_rc(const char *action, int rc, const char *stdout_data)
     /*  */
 643 {
 644     const char *stop_pattern[] = { "*stopped*", "*not*running*" };
 645     const char *running_pattern[] = { "*running*", "*OK*" };
 646     char *lower_std_output = NULL;
 647     int result;
 648 
 649 
 650     if (rc < 0) {
 651         return PCMK_OCF_UNKNOWN_ERROR;
 652     }
 653 
 654     /* Treat class heartbeat the same as class lsb. */
 655     if (!safe_str_eq(action, "status") && !safe_str_eq(action, "monitor")) {
 656         return services_get_ocf_exitcode(action, rc);
 657     }
 658 
 659     /* for status though, exit code is ignored,
 660      * and the stdout is scanned for specific strings */
 661     if (stdout_data == NULL) {
 662         crm_warn("No status output from the (hb) resource agent, assuming stopped");
 663         return PCMK_OCF_NOT_RUNNING;
 664     }
 665 
 666     lower_std_output = g_ascii_strdown(stdout_data, -1);
 667 
 668     if (pattern_matched(stop_pattern[0], lower_std_output) ||
 669         pattern_matched(stop_pattern[1], lower_std_output)) {
 670         result = PCMK_OCF_NOT_RUNNING;
 671     } else if (pattern_matched(running_pattern[0], lower_std_output) ||
 672         pattern_matched(running_pattern[1], stdout_data)) {
 673             /* "OK" is matched case sensitive */
 674         result = PCMK_OCF_OK;
 675     } else {
 676         /* It didn't say it was running - must be stopped */
 677         crm_debug("RA output did not match any pattern, assuming stopped");
 678         result = PCMK_OCF_NOT_RUNNING;
 679     }
 680     free(lower_std_output);
 681     return result;
 682 }
 683 #endif
 684 
 685 static int
 686 ocf2uniform_rc(int rc)
     /*  */
 687 {
 688     if (rc < 0 || rc > PCMK_OCF_FAILED_MASTER) {
 689         return PCMK_OCF_UNKNOWN_ERROR;
 690     }
 691 
 692     return rc;
 693 }
 694 
 695 static int
 696 stonith2uniform_rc(const char *action, int rc)
     /*  */
 697 {
 698     if (rc == -ENODEV) {
 699         if (safe_str_eq(action, "stop")) {
 700             rc = PCMK_OCF_OK;
 701         } else if (safe_str_eq(action, "start")) {
 702             rc = PCMK_OCF_NOT_INSTALLED;
 703         } else {
 704             rc = PCMK_OCF_NOT_RUNNING;
 705         }
 706     } else if (rc != 0) {
 707         rc = PCMK_OCF_UNKNOWN_ERROR;
 708     }
 709     return rc;
 710 }
 711 
 712 #if SUPPORT_NAGIOS
 713 static int
 714 nagios2uniform_rc(const char *action, int rc)
     /*  */
 715 {
 716     if (rc < 0) {
 717         return PCMK_OCF_UNKNOWN_ERROR;
 718     }
 719 
 720     switch (rc) {
 721         case NAGIOS_STATE_OK:
 722             return PCMK_OCF_OK;
 723         case NAGIOS_INSUFFICIENT_PRIV:
 724             return PCMK_OCF_INSUFFICIENT_PRIV;
 725         case NAGIOS_NOT_INSTALLED:
 726             return PCMK_OCF_NOT_INSTALLED;
 727         case NAGIOS_STATE_WARNING:
 728         case NAGIOS_STATE_CRITICAL:
 729         case NAGIOS_STATE_UNKNOWN:
 730         case NAGIOS_STATE_DEPENDENT:
 731         default:
 732             return PCMK_OCF_UNKNOWN_ERROR;
 733     }
 734 
 735     return PCMK_OCF_UNKNOWN_ERROR;
 736 }
 737 #endif
 738 
 739 static int
 740 get_uniform_rc(const char *standard, const char *action, int rc)
     /*  */
 741 {
 742     if (safe_str_eq(standard, PCMK_RESOURCE_CLASS_OCF)) {
 743         return ocf2uniform_rc(rc);
 744     } else if (safe_str_eq(standard, PCMK_RESOURCE_CLASS_STONITH)) {
 745         return stonith2uniform_rc(action, rc);
 746     } else if (safe_str_eq(standard, PCMK_RESOURCE_CLASS_SYSTEMD)) {
 747         return rc;
 748     } else if (safe_str_eq(standard, PCMK_RESOURCE_CLASS_UPSTART)) {
 749         return rc;
 750 #if SUPPORT_NAGIOS
 751     } else if (safe_str_eq(standard, PCMK_RESOURCE_CLASS_NAGIOS)) {
 752         return nagios2uniform_rc(action, rc);
 753 #endif
 754     } else {
 755         return services_get_ocf_exitcode(action, rc);
 756     }
 757 }
 758 
 759 static int
 760 action_get_uniform_rc(svc_action_t * action)
     /*  */
 761 {
 762     lrmd_cmd_t *cmd = action->cb_data;
 763 #if SUPPORT_HEARTBEAT
 764     if (safe_str_eq(action->standard, PCMK_RESOURCE_CLASS_HB)) {
 765         return hb2uniform_rc(cmd->action, action->rc, action->stdout_data);
 766     }
 767 #endif
 768     return get_uniform_rc(action->standard, cmd->action, action->rc);
 769 }
 770 
 771 void
 772 notify_of_new_client(crm_client_t *new_client)
     /*  */
 773 {
 774     crm_client_t *client = NULL;
 775     GHashTableIter iter;
 776     xmlNode *notify = NULL;
 777     char *key = NULL;
 778 
 779     notify = create_xml_node(NULL, T_LRMD_NOTIFY);
 780     crm_xml_add(notify, F_LRMD_ORIGIN, __FUNCTION__);
 781     crm_xml_add(notify, F_LRMD_OPERATION, LRMD_OP_NEW_CLIENT);
 782 
 783     g_hash_table_iter_init(&iter, client_connections);
 784     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & client)) {
 785 
 786         if (safe_str_eq(client->id, new_client->id)) {
 787             continue;
 788         }
 789 
 790         send_client_notify((gpointer) key, (gpointer) client, (gpointer) notify);
 791     }
 792     free_xml(notify);
 793 }
 794 
 795 static char *
 796 parse_exit_reason(const char *output)
     /*  */
 797 {
 798     const char *cur = NULL;
 799     const char *last = NULL;
 800     char *reason = NULL;
 801     static int cookie_len = 0;
 802     char *eol = NULL;
 803 
 804     if (output == NULL) {
 805         return NULL;
 806     }
 807 
 808     if (!cookie_len) {
 809         cookie_len = strlen(PCMK_OCF_REASON_PREFIX);
 810     }
 811 
 812     cur = strstr(output, PCMK_OCF_REASON_PREFIX);
 813     for (; cur != NULL; cur = strstr(cur, PCMK_OCF_REASON_PREFIX)) {
 814         /* skip over the cookie delimiter string */
 815         cur += cookie_len;
 816         last = cur;
 817     }
 818     if (last == NULL) {
 819         return NULL;
 820     }
 821 
 822     /* make our own copy */
 823     reason = calloc(1, (EXIT_REASON_MAX_LEN+1));
 824     CRM_ASSERT(reason);
 825 
 826     /* limit reason string size */
 827     strncpy(reason, last, EXIT_REASON_MAX_LEN);
 828 
 829     /* truncate everything after a new line */
 830     eol = strchr(reason, '\n');
 831     if (eol != NULL) {
 832         *eol = '\0';
 833     }
 834 
 835     return reason;
 836 }
 837 
 838 void
 839 client_disconnect_cleanup(const char *client_id)
     /*  */
 840 {
 841     GHashTableIter iter;
 842     lrmd_rsc_t *rsc = NULL;
 843     char *key = NULL;
 844 
 845     g_hash_table_iter_init(&iter, rsc_list);
 846     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
 847         if (rsc->call_opts & lrmd_opt_drop_recurring) {
 848             /* This client is disconnecting, drop any recurring operations
 849              * it may have initiated on the resource */
 850             cancel_all_recurring(rsc, client_id);
 851         }
 852     }
 853 }
 854 
 855 static void
 856 action_complete(svc_action_t * action)
     /*  */
 857 {
 858     lrmd_rsc_t *rsc;
 859     lrmd_cmd_t *cmd = action->cb_data;
 860     const char *rclass = NULL;
 861 
 862     bool goagain = false;
 863 
 864     if (!cmd) {
 865         crm_err("LRMD action (%s) completed does not match any known operations.", action->id);
 866         return;
 867     }
 868 #ifdef HAVE_SYS_TIMEB_H
 869     if (cmd->exec_rc != action->rc) {
 870         ftime(&cmd->t_rcchange);
 871     }
 872 #endif
 873 
 874     cmd->last_pid = action->pid;
 875     cmd->exec_rc = action_get_uniform_rc(action);
 876     cmd->lrmd_op_status = action->status;
 877     rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL;
 878 
 879     if (rsc && safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_SERVICE)) {
 880         rclass = resources_find_service_class(rsc->class);
 881     } else if(rsc) {
 882         rclass = rsc->class;
 883     }
 884 
 885     if (safe_str_eq(rclass, PCMK_RESOURCE_CLASS_SYSTEMD)) {
 886         if(cmd->exec_rc == PCMK_OCF_OK && safe_str_eq(cmd->action, "start")) {
 887             /* systemd I curse thee!
 888              *
 889              * systemd returns from start actions after the start _begins_
 890              * not after it completes.
 891              *
 892              * So we have to jump through a few hoops so that we don't
 893              * report 'complete' to the rest of pacemaker until, you know,
 894              * it's actually done.
 895              */
 896             goagain = true;
 897             cmd->real_action = cmd->action;
 898             cmd->action = strdup("monitor");
 899 
 900         } else if(cmd->exec_rc == PCMK_OCF_OK && safe_str_eq(cmd->action, "stop")) {
 901             goagain = true;
 902             cmd->real_action = cmd->action;
 903             cmd->action = strdup("monitor");
 904 
 905         } else if(cmd->real_action) {
 906             /* Ok, so this is the follow up monitor action to check if start actually completed */
 907             if(cmd->lrmd_op_status == PCMK_LRM_OP_DONE && cmd->exec_rc == PCMK_OCF_PENDING) {
 908                 goagain = true;
 909             } else if(cmd->exec_rc == PCMK_OCF_OK && safe_str_eq(cmd->real_action, "stop")) {
 910                 goagain = true;
 911 
 912             } else {
 913 #ifdef HAVE_SYS_TIMEB_H
 914                 int time_sum = time_diff_ms(NULL, &cmd->t_first_run);
 915                 int timeout_left = cmd->timeout_orig - time_sum;
 916 
 917                 crm_debug("%s %s is now complete (elapsed=%dms, remaining=%dms): %s (%d)",
 918                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left, services_ocf_exitcode_str(cmd->exec_rc), cmd->exec_rc);
 919                 cmd_original_times(cmd);
 920 #endif
 921 
 922                 if(cmd->lrmd_op_status == PCMK_LRM_OP_DONE && cmd->exec_rc == PCMK_OCF_NOT_RUNNING && safe_str_eq(cmd->real_action, "stop")) {
 923                     cmd->exec_rc = PCMK_OCF_OK;
 924                 }
 925             }
 926         }
 927     }
 928 
 929 #if SUPPORT_NAGIOS
 930     if (rsc && safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS)) {
 931         if (safe_str_eq(cmd->action, "monitor") &&
 932             cmd->interval == 0 && cmd->exec_rc == PCMK_OCF_OK) {
 933             /* Successfully executed --version for the nagios plugin */
 934             cmd->exec_rc = PCMK_OCF_NOT_RUNNING;
 935 
 936         } else if (safe_str_eq(cmd->action, "start") && cmd->exec_rc != PCMK_OCF_OK) {
 937             goagain = true;
 938         }
 939     }
 940 #endif
 941 
 942     /* Wrapping this section in ifdef implies that systemd resources are not
 943      * fully supported on platforms without sys/timeb.h. Since timeb is
 944      * obsolete, we should eventually prefer a clock_gettime() implementation
 945      * (wrapped in its own ifdef) with timeb as a fallback.
 946      */
 947 #ifdef HAVE_SYS_TIMEB_H
 948     if(goagain) {
 949         int time_sum = time_diff_ms(NULL, &cmd->t_first_run);
 950         int timeout_left = cmd->timeout_orig - time_sum;
 951         int delay = cmd->timeout_orig / 10;
 952 
 953         if(delay >= timeout_left && timeout_left > 20) {
 954             delay = timeout_left/2;
 955         }
 956 
 957         delay = QB_MIN(2000, delay);
 958         if (delay < timeout_left) {
 959             cmd->start_delay = delay;
 960             cmd->timeout = timeout_left;
 961 
 962             if(cmd->exec_rc == PCMK_OCF_OK) {
 963                 crm_debug("%s %s may still be in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
 964                           cmd->rsc_id, cmd->real_action, time_sum, timeout_left, delay);
 965 
 966             } else if(cmd->exec_rc == PCMK_OCF_PENDING) {
 967                 crm_info("%s %s is still in progress: re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
 968                          cmd->rsc_id, cmd->action, time_sum, timeout_left, delay);
 969 
 970             } else {
 971                 crm_notice("%s %s failed '%s' (%d): re-scheduling (elapsed=%dms, remaining=%dms, start_delay=%dms)",
 972                            cmd->rsc_id, cmd->action, services_ocf_exitcode_str(cmd->exec_rc), cmd->exec_rc, time_sum, timeout_left, delay);
 973             }
 974 
 975             cmd_reset(cmd);
 976             if(rsc) {
 977                 rsc->active = NULL;
 978             }
 979             schedule_lrmd_cmd(rsc, cmd);
 980 
 981             /* Don't finalize cmd, we're not done with it yet */
 982             return;
 983 
 984         } else {
 985             crm_notice("Giving up on %s %s (rc=%d): timeout (elapsed=%dms, remaining=%dms)",
 986                        cmd->rsc_id, cmd->real_action?cmd->real_action:cmd->action, cmd->exec_rc, time_sum, timeout_left);
 987             cmd->lrmd_op_status = PCMK_LRM_OP_TIMEOUT;
 988             cmd->exec_rc = PCMK_OCF_TIMEOUT;
 989             cmd_original_times(cmd);
 990         }
 991     }
 992 #endif
 993 
 994     if (action->stderr_data) {
 995         cmd->output = strdup(action->stderr_data);
 996         cmd->exit_reason = parse_exit_reason(action->stderr_data);
 997 
 998     } else if (action->stdout_data) {
 999         cmd->output = strdup(action->stdout_data);
1000     }
1001 
1002     cmd_finalize(cmd, rsc);
1003 }
1004 
1005 static void
1006 stonith_action_complete(lrmd_cmd_t * cmd, int rc)
     /*  */
1007 {
1008     int recurring = cmd->interval;
1009     lrmd_rsc_t *rsc = NULL;
1010 
1011     cmd->exec_rc = get_uniform_rc(PCMK_RESOURCE_CLASS_STONITH, cmd->action, rc);
1012 
1013     rsc = g_hash_table_lookup(rsc_list, cmd->rsc_id);
1014 
1015     if (cmd->lrmd_op_status == PCMK_LRM_OP_CANCELLED) {
1016         recurring = 0;
1017         /* do nothing */
1018 
1019     } else if (rc == -ENODEV && safe_str_eq(cmd->action, "monitor")) {
1020         /* Not registered == inactive */
1021         cmd->lrmd_op_status = PCMK_LRM_OP_DONE;
1022         cmd->exec_rc = PCMK_OCF_NOT_RUNNING;
1023 
1024     } else if (rc) {
1025         /* Attempt to map return codes to op status if possible */
1026         switch (rc) {
1027             case -EPROTONOSUPPORT:
1028                 cmd->lrmd_op_status = PCMK_LRM_OP_NOTSUPPORTED;
1029                 break;
1030             case -ETIME:
1031                 cmd->lrmd_op_status = PCMK_LRM_OP_TIMEOUT;
1032                 break;
1033             default:
1034                 /* TODO: This looks wrong.  Status should be _DONE and exec_rc set to an error */
1035                 cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
1036         }
1037     } else {
1038         /* command successful */
1039         cmd->lrmd_op_status = PCMK_LRM_OP_DONE;
1040         if (safe_str_eq(cmd->action, "start") && rsc) {
1041             rsc->stonith_started = 1;
1042         }
1043     }
1044 
1045     if (recurring && rsc) {
1046         if (cmd->stonith_recurring_id) {
1047             g_source_remove(cmd->stonith_recurring_id);
1048         }
1049         cmd->stonith_recurring_id = g_timeout_add(cmd->interval, stonith_recurring_op_helper, cmd);
1050     }
1051 
1052     cmd_finalize(cmd, rsc);
1053 }
1054 
1055 static void
1056 lrmd_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
     /*  */
1057 {
1058     stonith_action_complete(data->userdata, data->rc);
1059 }
1060 
1061 void
1062 stonith_connection_failed(void)
     /*  */
1063 {
1064     GHashTableIter iter;
1065     GList *cmd_list = NULL;
1066     GList *cmd_iter = NULL;
1067     lrmd_rsc_t *rsc = NULL;
1068     char *key = NULL;
1069 
1070     g_hash_table_iter_init(&iter, rsc_list);
1071     while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & rsc)) {
1072         if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) {
1073             if (rsc->active) {
1074                 cmd_list = g_list_append(cmd_list, rsc->active);
1075             }
1076             if (rsc->recurring_ops) {
1077                 cmd_list = g_list_concat(cmd_list, rsc->recurring_ops);
1078             }
1079             if (rsc->pending_ops) {
1080                 cmd_list = g_list_concat(cmd_list, rsc->pending_ops);
1081             }
1082             rsc->pending_ops = rsc->recurring_ops = NULL;
1083         }
1084     }
1085 
1086     if (!cmd_list) {
1087         return;
1088     }
1089 
1090     crm_err("STONITH connection failed, finalizing %d pending operations.",
1091             g_list_length(cmd_list));
1092     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
1093         stonith_action_complete(cmd_iter->data, -ENOTCONN);
1094     }
1095     g_list_free(cmd_list);
1096 }
1097 
1098 static int
1099 lrmd_rsc_execute_stonith(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
     /*  */
1100 {
1101     int rc = 0;
1102     int do_monitor = 0;
1103 
1104     stonith_t *stonith_api = get_stonith_connection();
1105 
1106     if (!stonith_api) {
1107         cmd->exec_rc = get_uniform_rc(PCMK_RESOURCE_CLASS_STONITH, cmd->action,
1108                                       -ENOTCONN);
1109         cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
1110         cmd_finalize(cmd, rsc);
1111         return -EUNATCH;
1112     }
1113 
1114     if (safe_str_eq(cmd->action, "start")) {
1115         char *key = NULL;
1116         char *value = NULL;
1117         stonith_key_value_t *device_params = NULL;
1118 
1119         if (cmd->params) {
1120             GHashTableIter iter;
1121 
1122             g_hash_table_iter_init(&iter, cmd->params);
1123             while (g_hash_table_iter_next(&iter, (gpointer *) & key, (gpointer *) & value)) {
1124                 device_params = stonith_key_value_add(device_params, key, value);
1125             }
1126         }
1127 
1128         /* Stonith automatically registers devices from the IPC when changes occur,
1129          * but to avoid a possible race condition between stonith receiving the IPC update
1130          * and the lrmd requesting that resource, the lrmd still registers the device as well.
1131          * Stonith knows how to handle duplicate device registrations correctly. */
1132         rc = stonith_api->cmds->register_device(stonith_api,
1133                                                 st_opt_sync_call,
1134                                                 cmd->rsc_id,
1135                                                 rsc->provider, rsc->type, device_params);
1136 
1137         stonith_key_value_freeall(device_params, 1, 1);
1138         if (rc == 0) {
1139             do_monitor = 1;
1140         }
1141     } else if (safe_str_eq(cmd->action, "stop")) {
1142         rc = stonith_api->cmds->remove_device(stonith_api, st_opt_sync_call, cmd->rsc_id);
1143         rsc->stonith_started = 0;
1144     } else if (safe_str_eq(cmd->action, "monitor")) {
1145         if (cmd->interval) {
1146             do_monitor = 1;
1147         } else {
1148             rc = rsc->stonith_started ? 0 : -ENODEV;
1149         }
1150     }
1151 
1152     if (!do_monitor) {
1153         goto cleanup_stonith_exec;
1154     }
1155 
1156     rc = stonith_api->cmds->monitor(stonith_api, 0, cmd->rsc_id, cmd->timeout / 1000);
1157 
1158     rc = stonith_api->cmds->register_callback(stonith_api,
1159                                               rc,
1160                                               0,
1161                                               0,
1162                                               cmd, "lrmd_stonith_callback", lrmd_stonith_callback);
1163 
1164     /* don't cleanup yet, we will find out the result of the monitor later */
1165     if (rc > 0) {
1166         rsc->active = cmd;
1167         return rc;
1168     } else if (rc == 0) {
1169         rc = -1;
1170     }
1171 
1172   cleanup_stonith_exec:
1173     stonith_action_complete(cmd, rc);
1174     return rc;
1175 }
1176 
1177 static int
1178 lrmd_rsc_execute_service_lib(lrmd_rsc_t * rsc, lrmd_cmd_t * cmd)
     /*  */
1179 {
1180     svc_action_t *action = NULL;
1181     GHashTable *params_copy = NULL;
1182 
1183     CRM_ASSERT(rsc);
1184     CRM_ASSERT(cmd);
1185 
1186     crm_trace("Creating action, resource:%s action:%s class:%s provider:%s agent:%s",
1187               rsc->rsc_id, cmd->action, rsc->class, rsc->provider, rsc->type);
1188 
1189 #if SUPPORT_NAGIOS
1190     /* Recurring operations are cancelled anyway for a stop operation */
1191     if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_NAGIOS)
1192         && safe_str_eq(cmd->action, "stop")) {
1193 
1194         cmd->exec_rc = PCMK_OCF_OK;
1195         goto exec_done;
1196     }
1197 #endif
1198 
1199     params_copy = crm_str_table_dup(cmd->params);
1200 
1201     if (cmd->isolation_wrapper) {
1202         g_hash_table_remove(params_copy, "CRM_meta_isolation_wrapper");
1203         action = resources_action_create(rsc->rsc_id,
1204                                          PCMK_RESOURCE_CLASS_OCF,
1205                                          LRMD_ISOLATION_PROVIDER,
1206                                          cmd->isolation_wrapper,
1207                                          cmd->action, /*action will be normalized in wrapper*/
1208                                          cmd->interval,
1209                                          cmd->timeout,
1210                                          params_copy,
1211                                          cmd->service_flags);
1212     } else {
1213         action = resources_action_create(rsc->rsc_id,
1214                                          rsc->class,
1215                                          rsc->provider,
1216                                          rsc->type,
1217                                          normalize_action_name(rsc, cmd->action),
1218                                          cmd->interval,
1219                                          cmd->timeout,
1220                                          params_copy,
1221                                          cmd->service_flags);
1222     }
1223 
1224     if (!action) {
1225         crm_err("Failed to create action, action:%s on resource %s", cmd->action, rsc->rsc_id);
1226         cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
1227         goto exec_done;
1228     }
1229 
1230     action->cb_data = cmd;
1231 
1232     /* 'cmd' may not be valid after this point if
1233      * services_action_async() returned TRUE
1234      *
1235      * Upstart and systemd both synchronously determine monitor/status
1236      * results and call action_complete (which may free 'cmd') if necessary.
1237      */
1238     if (services_action_async(action, action_complete)) {
1239         return TRUE;
1240     }
1241 
1242     cmd->exec_rc = action->rc;
1243     if(action->status != PCMK_LRM_OP_DONE) {
1244         cmd->lrmd_op_status = action->status;
1245     } else {
1246         cmd->lrmd_op_status = PCMK_LRM_OP_ERROR;
1247     }
1248     services_action_free(action);
1249     action = NULL;
1250 
1251   exec_done:
1252     cmd_finalize(cmd, rsc);
1253     return TRUE;
1254 }
1255 
1256 static gboolean
1257 lrmd_rsc_execute(lrmd_rsc_t * rsc)
     /*  */
1258 {
1259     lrmd_cmd_t *cmd = NULL;
1260 
1261     CRM_CHECK(rsc != NULL, return FALSE);
1262 
1263     if (rsc->active) {
1264         crm_trace("%s is still active", rsc->rsc_id);
1265         return TRUE;
1266     }
1267 
1268     if (rsc->pending_ops) {
1269         GList *first = rsc->pending_ops;
1270 
1271         cmd = first->data;
1272         if (cmd->delay_id) {
1273             crm_trace
1274                 ("Command %s %s was asked to run too early, waiting for start_delay timeout of %dms",
1275                  cmd->rsc_id, cmd->action, cmd->start_delay);
1276             return TRUE;
1277         }
1278         rsc->pending_ops = g_list_remove_link(rsc->pending_ops, first);
1279         g_list_free_1(first);
1280 
1281 #ifdef HAVE_SYS_TIMEB_H
1282         if (cmd->t_first_run.time == 0) {
1283             ftime(&cmd->t_first_run);
1284         }
1285         ftime(&cmd->t_run);
1286 #endif
1287     }
1288 
1289     if (!cmd) {
1290         crm_trace("Nothing further to do for %s", rsc->rsc_id);
1291         return TRUE;
1292     }
1293 
1294     rsc->active = cmd;          /* only one op at a time for a rsc */
1295     if (cmd->interval) {
1296         rsc->recurring_ops = g_list_append(rsc->recurring_ops, cmd);
1297     }
1298 
1299     log_execute(cmd);
1300 
1301     if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) {
1302         lrmd_rsc_execute_stonith(rsc, cmd);
1303     } else {
1304         lrmd_rsc_execute_service_lib(rsc, cmd);
1305     }
1306 
1307     return TRUE;
1308 }
1309 
1310 static gboolean
1311 lrmd_rsc_dispatch(gpointer user_data)
     /*  */
1312 {
1313     return lrmd_rsc_execute(user_data);
1314 }
1315 
1316 void
1317 free_rsc(gpointer data)
     /*  */
1318 {
1319     GListPtr gIter = NULL;
1320     lrmd_rsc_t *rsc = data;
1321     int is_stonith = safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH);
1322 
1323     gIter = rsc->pending_ops;
1324     while (gIter != NULL) {
1325         GListPtr next = gIter->next;
1326         lrmd_cmd_t *cmd = gIter->data;
1327 
1328         /* command was never executed */
1329         cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
1330         cmd_finalize(cmd, NULL);
1331 
1332         gIter = next;
1333     }
1334     /* frees list, but not list elements. */
1335     g_list_free(rsc->pending_ops);
1336 
1337     gIter = rsc->recurring_ops;
1338     while (gIter != NULL) {
1339         GListPtr next = gIter->next;
1340         lrmd_cmd_t *cmd = gIter->data;
1341 
1342         if (is_stonith) {
1343             cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
1344             /* If a stonith command is in-flight, just mark it as cancelled;
1345              * it is not safe to finalize/free the cmd until the stonith api
1346              * says it has either completed or timed out.
1347              */
1348             if (rsc->active != cmd) {
1349                 cmd_finalize(cmd, NULL);
1350             }
1351         } else {
1352             /* This command is already handed off to service library,
1353              * let service library cancel it and tell us via the callback
1354              * when it is cancelled. The rsc can be safely destroyed
1355              * even if we are waiting for the cancel result */
1356             services_action_cancel(rsc->rsc_id, normalize_action_name(rsc, cmd->action), cmd->interval);
1357         }
1358 
1359         gIter = next;
1360     }
1361     /* frees list, but not list elements. */
1362     g_list_free(rsc->recurring_ops);
1363 
1364     free(rsc->rsc_id);
1365     free(rsc->class);
1366     free(rsc->provider);
1367     free(rsc->type);
1368     mainloop_destroy_trigger(rsc->work);
1369 
1370     free(rsc);
1371 }
1372 
1373 static int
1374 process_lrmd_signon(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1375 {
1376     xmlNode *reply = create_xml_node(NULL, "reply");
1377     const char *is_ipc_provider = crm_element_value(request, F_LRMD_IS_IPC_PROVIDER);
1378     const char *protocol_version = crm_element_value(request, F_LRMD_PROTOCOL_VERSION);
1379 
1380     if (compare_version(protocol_version, LRMD_MIN_PROTOCOL_VERSION) < 0) {
1381         crm_err("Cluster API version must be greater than or equal to %s, not %s",
1382                 LRMD_MIN_PROTOCOL_VERSION, protocol_version);
1383         crm_xml_add_int(reply, F_LRMD_RC, -EPROTO);
1384     }
1385 
1386     crm_xml_add(reply, F_LRMD_OPERATION, CRM_OP_REGISTER);
1387     crm_xml_add(reply, F_LRMD_CLIENTID, client->id);
1388     crm_xml_add(reply, F_LRMD_PROTOCOL_VERSION, LRMD_PROTOCOL_VERSION);
1389     lrmd_server_send_reply(client, id, reply);
1390 
1391     if (crm_is_true(is_ipc_provider)) {
1392         /* this is a remote connection from a cluster nodes crmd */
1393 #ifdef SUPPORT_REMOTE
1394         ipc_proxy_add_provider(client);
1395 #endif
1396     }
1397 
1398     free_xml(reply);
1399     return pcmk_ok;
1400 }
1401 
1402 static int
1403 process_lrmd_rsc_register(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1404 {
1405     int rc = pcmk_ok;
1406     lrmd_rsc_t *rsc = build_rsc_from_xml(request);
1407     lrmd_rsc_t *dup = g_hash_table_lookup(rsc_list, rsc->rsc_id);
1408 
1409     if (dup &&
1410         safe_str_eq(rsc->class, dup->class) &&
1411         safe_str_eq(rsc->provider, dup->provider) && safe_str_eq(rsc->type, dup->type)) {
1412 
1413         crm_warn("Can't add, RSC '%s' already present in the rsc list (%d active resources)",
1414                  rsc->rsc_id, g_hash_table_size(rsc_list));
1415 
1416         free_rsc(rsc);
1417         return rc;
1418     }
1419 
1420     g_hash_table_replace(rsc_list, rsc->rsc_id, rsc);
1421     crm_info("Added '%s' to the rsc list (%d active resources)",
1422              rsc->rsc_id, g_hash_table_size(rsc_list));
1423 
1424     return rc;
1425 }
1426 
1427 static void
1428 process_lrmd_get_rsc_info(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1429 {
1430     int rc = pcmk_ok;
1431     int send_rc = 0;
1432     int call_id = 0;
1433     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
1434     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1435     xmlNode *reply = NULL;
1436     lrmd_rsc_t *rsc = NULL;
1437 
1438     crm_element_value_int(request, F_LRMD_CALLID, &call_id);
1439 
1440     if (!rsc_id) {
1441         rc = -ENODEV;
1442         goto get_rsc_done;
1443     }
1444 
1445     if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
1446         crm_info("Resource '%s' not found (%d active resources)",
1447                  rsc_id, g_hash_table_size(rsc_list));
1448         rc = -ENODEV;
1449         goto get_rsc_done;
1450     }
1451 
1452   get_rsc_done:
1453 
1454     reply = create_xml_node(NULL, T_LRMD_REPLY);
1455     crm_xml_add(reply, F_LRMD_ORIGIN, __FUNCTION__);
1456     crm_xml_add_int(reply, F_LRMD_RC, rc);
1457     crm_xml_add_int(reply, F_LRMD_CALLID, call_id);
1458 
1459     if (rsc) {
1460         crm_xml_add(reply, F_LRMD_RSC_ID, rsc->rsc_id);
1461         crm_xml_add(reply, F_LRMD_CLASS, rsc->class);
1462         crm_xml_add(reply, F_LRMD_PROVIDER, rsc->provider);
1463         crm_xml_add(reply, F_LRMD_TYPE, rsc->type);
1464     }
1465 
1466     send_rc = lrmd_server_send_reply(client, id, reply);
1467 
1468     if (send_rc < 0) {
1469         crm_warn("LRMD reply to %s failed: %d", client->name, send_rc);
1470     }
1471 
1472     free_xml(reply);
1473 }
1474 
1475 static int
1476 process_lrmd_rsc_unregister(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1477 {
1478     int rc = pcmk_ok;
1479     lrmd_rsc_t *rsc = NULL;
1480     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
1481     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1482 
1483     if (!rsc_id) {
1484         return -ENODEV;
1485     }
1486 
1487     if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
1488         crm_info("Resource '%s' not found (%d active resources)",
1489                  rsc_id, g_hash_table_size(rsc_list));
1490         return pcmk_ok;
1491     }
1492 
1493     if (rsc->active) {
1494         /* let the caller know there are still active ops on this rsc to watch for */
1495         crm_trace("Operation still in progress: %p", rsc->active);
1496         rc = -EINPROGRESS;
1497     }
1498 
1499     g_hash_table_remove(rsc_list, rsc_id);
1500 
1501     return rc;
1502 }
1503 
1504 static int
1505 process_lrmd_rsc_exec(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1506 {
1507     lrmd_rsc_t *rsc = NULL;
1508     lrmd_cmd_t *cmd = NULL;
1509     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
1510     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1511     int call_id;
1512 
1513     if (!rsc_id) {
1514         return -EINVAL;
1515     }
1516     if (!(rsc = g_hash_table_lookup(rsc_list, rsc_id))) {
1517         crm_info("Resource '%s' not found (%d active resources)",
1518                  rsc_id, g_hash_table_size(rsc_list));
1519         return -ENODEV;
1520     }
1521 
1522     cmd = create_lrmd_cmd(request, client, rsc);
1523     call_id = cmd->call_id;
1524 
1525     /* Don't reference cmd after handing it off to be scheduled.
1526      * The cmd could get merged and freed. */
1527     schedule_lrmd_cmd(rsc, cmd);
1528 
1529     return call_id;
1530 }
1531 
1532 static int
1533 cancel_op(const char *rsc_id, const char *action, int interval)
     /*  */
1534 {
1535     GListPtr gIter = NULL;
1536     lrmd_rsc_t *rsc = g_hash_table_lookup(rsc_list, rsc_id);
1537 
1538     /* How to cancel an action.
1539      * 1. Check pending ops list, if it hasn't been handed off
1540      *    to the service library or stonith recurring list remove
1541      *    it there and that will stop it.
1542      * 2. If it isn't in the pending ops list, then it's either a
1543      *    recurring op in the stonith recurring list, or the service
1544      *    library's recurring list.  Stop it there
1545      * 3. If not found in any lists, then this operation has either
1546      *    been executed already and is not a recurring operation, or
1547      *    never existed.
1548      */
1549     if (!rsc) {
1550         return -ENODEV;
1551     }
1552 
1553     for (gIter = rsc->pending_ops; gIter != NULL; gIter = gIter->next) {
1554         lrmd_cmd_t *cmd = gIter->data;
1555 
1556         if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
1557             cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
1558             cmd_finalize(cmd, rsc);
1559             return pcmk_ok;
1560         }
1561     }
1562 
1563     if (safe_str_eq(rsc->class, PCMK_RESOURCE_CLASS_STONITH)) {
1564         /* The service library does not handle stonith operations.
1565          * We have to handle recurring stonith operations ourselves. */
1566         for (gIter = rsc->recurring_ops; gIter != NULL; gIter = gIter->next) {
1567             lrmd_cmd_t *cmd = gIter->data;
1568 
1569             if (safe_str_eq(cmd->action, action) && cmd->interval == interval) {
1570                 cmd->lrmd_op_status = PCMK_LRM_OP_CANCELLED;
1571                 if (rsc->active != cmd) {
1572                     cmd_finalize(cmd, rsc);
1573                 }
1574                 return pcmk_ok;
1575             }
1576         }
1577     } else if (services_action_cancel(rsc_id, normalize_action_name(rsc, action), interval) == TRUE) {
1578         /* The service library will tell the action_complete callback function
1579          * this action was cancelled, which will destroy the cmd and remove
1580          * it from the recurring_op list. Do not do that in this function
1581          * if the service library says it cancelled it. */
1582         return pcmk_ok;
1583     }
1584 
1585     return -EOPNOTSUPP;
1586 }
1587 
1588 static void
1589 cancel_all_recurring(lrmd_rsc_t * rsc, const char *client_id)
     /*  */
1590 {
1591     GList *cmd_list = NULL;
1592     GList *cmd_iter = NULL;
1593 
1594     /* Notice a copy of each list is created when concat is called.
1595      * This prevents odd behavior from occurring when the cmd_list
1596      * is iterated through later on.  It is possible the cancel_op
1597      * function may end up modifying the recurring_ops and pending_ops
1598      * lists.  If we did not copy those lists, our cmd_list iteration
1599      * could get messed up.*/
1600     if (rsc->recurring_ops) {
1601         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->recurring_ops));
1602     }
1603     if (rsc->pending_ops) {
1604         cmd_list = g_list_concat(cmd_list, g_list_copy(rsc->pending_ops));
1605     }
1606     if (!cmd_list) {
1607         return;
1608     }
1609 
1610     for (cmd_iter = cmd_list; cmd_iter; cmd_iter = cmd_iter->next) {
1611         lrmd_cmd_t *cmd = cmd_iter->data;
1612 
1613         if (cmd->interval == 0) {
1614             continue;
1615         }
1616 
1617         if (client_id && safe_str_neq(cmd->client_id, client_id)) {
1618             continue;
1619         }
1620 
1621         cancel_op(rsc->rsc_id, cmd->action, cmd->interval);
1622     }
1623     /* frees only the copied list data, not the cmds */
1624     g_list_free(cmd_list);
1625 }
1626 
1627 static int
1628 process_lrmd_rsc_cancel(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1629 {
1630     xmlNode *rsc_xml = get_xpath_object("//" F_LRMD_RSC, request, LOG_ERR);
1631     const char *rsc_id = crm_element_value(rsc_xml, F_LRMD_RSC_ID);
1632     const char *action = crm_element_value(rsc_xml, F_LRMD_RSC_ACTION);
1633     int interval = 0;
1634 
1635     crm_element_value_int(rsc_xml, F_LRMD_RSC_INTERVAL, &interval);
1636 
1637     if (!rsc_id || !action) {
1638         return -EINVAL;
1639     }
1640 
1641     return cancel_op(rsc_id, action, interval);
1642 }
1643 
1644 void
1645 process_lrmd_message(crm_client_t * client, uint32_t id, xmlNode * request)
     /*  */
1646 {
1647     int rc = pcmk_ok;
1648     int call_id = 0;
1649     const char *op = crm_element_value(request, F_LRMD_OPERATION);
1650     int do_reply = 0;
1651     int do_notify = 0;
1652 
1653     crm_trace("Processing %s operation from %s", op, client->id);
1654     crm_element_value_int(request, F_LRMD_CALLID, &call_id);
1655 
1656     if (crm_str_eq(op, CRM_OP_IPC_FWD, TRUE)) {
1657 #ifdef SUPPORT_REMOTE
1658         ipc_proxy_forward_client(client, request);
1659 #endif
1660         do_reply = 1;
1661     } else if (crm_str_eq(op, CRM_OP_REGISTER, TRUE)) {
1662         rc = process_lrmd_signon(client, id, request);
1663     } else if (crm_str_eq(op, LRMD_OP_RSC_REG, TRUE)) {
1664         rc = process_lrmd_rsc_register(client, id, request);
1665         do_notify = 1;
1666         do_reply = 1;
1667     } else if (crm_str_eq(op, LRMD_OP_RSC_INFO, TRUE)) {
1668         process_lrmd_get_rsc_info(client, id, request);
1669     } else if (crm_str_eq(op, LRMD_OP_RSC_UNREG, TRUE)) {
1670         rc = process_lrmd_rsc_unregister(client, id, request);
1671         /* don't notify anyone about failed un-registers */
1672         if (rc == pcmk_ok || rc == -EINPROGRESS) {
1673             do_notify = 1;
1674         }
1675         do_reply = 1;
1676     } else if (crm_str_eq(op, LRMD_OP_RSC_EXEC, TRUE)) {
1677         rc = process_lrmd_rsc_exec(client, id, request);
1678         do_reply = 1;
1679     } else if (crm_str_eq(op, LRMD_OP_RSC_CANCEL, TRUE)) {
1680         rc = process_lrmd_rsc_cancel(client, id, request);
1681         do_reply = 1;
1682     } else if (crm_str_eq(op, LRMD_OP_POKE, TRUE)) {
1683         do_notify = 1;
1684         do_reply = 1;
1685     } else if (crm_str_eq(op, LRMD_OP_CHECK, TRUE)) {
1686         xmlNode *data = get_message_xml(request, F_LRMD_CALLDATA); 
1687         const char *timeout = crm_element_value(data, F_LRMD_WATCHDOG);
1688         CRM_LOG_ASSERT(data != NULL);
1689         check_sbd_timeout(timeout);
1690     } else if (crm_str_eq(op, LRMD_OP_ALERT_EXEC, TRUE)) {
1691         rc = process_lrmd_alert_exec(client, id, request);
1692         do_reply = 1;
1693     } else {
1694         rc = -EOPNOTSUPP;
1695         do_reply = 1;
1696         crm_err("Unknown %s from %s", op, client->name);
1697         crm_log_xml_warn(request, "UnknownOp");
1698     }
1699 
1700     crm_debug("Processed %s operation from %s: rc=%d, reply=%d, notify=%d",
1701               op, client->id, rc, do_reply, do_notify);
1702 
1703     if (do_reply) {
1704         send_reply(client, rc, id, call_id);
1705     }
1706 
1707     if (do_notify) {
1708         send_generic_notify(rc, request);
1709     }
1710 }

/* */