root/fencing/remote.c

/* */

DEFINITIONS

1. sort_strings
2. free_remote_query
3. count_peer_device
4. count_peer_devices
5. find_peer_device
6. grab_peer_device
7. clear_remote_op_timers
8. free_remote_op
9. op_requested_action
10. op_phase_off
11. op_phase_on
12. undo_op_remap
13. create_op_done_notify
14. bcast_result_to_peers
15. handle_local_reply_and_notify
16. handle_duplicates
17. remote_op_done
18. remote_op_watchdog_done
19. remote_op_timeout_one
20. remote_op_timeout
21. remote_op_query_timeout
22. topology_is_empty
23. add_required_device
24. remove_required_device
25. set_op_device_list
26. topology_matches
27. find_topology_for_host
28. stonith_topology_next
29. merge_duplicates
30. fencing_active_peers
31. stonith_manual_ack
32. stonith_get_peer_name
33. create_remote_stonith_op
34. initiate_remote_stonith_op
35. find_best_peer
36. stonith_choose_peer
37. get_device_timeout
38. add_device_timeout
39. get_peer_timeout
40. get_op_total_timeout
41. report_timeout_period
42. advance_op_topology
43. call_remote_stonith
44. sort_peers
45. all_topology_devices_found
46. parse_action_specific
47. add_device_properties
48. add_result
49. process_remote_stonith_query
50. process_remote_stonith_exec
51. stonith_fence_history
52. stonith_check_fence_tolerance

   1 /*
   2  * Copyright (C) 2009 Andrew Beekhof <andrew@beekhof.net>
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2 of the License, or (at your option) any later version.
   8  *
   9  * This software is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 
  19 #include <crm_internal.h>
  20 
  21 #include <sys/param.h>
  22 #include <stdio.h>
  23 #include <sys/types.h>
  24 #include <sys/wait.h>
  25 #include <sys/stat.h>
  26 #include <unistd.h>
  27 #include <sys/utsname.h>
  28 
  29 #include <stdlib.h>
  30 #include <errno.h>
  31 #include <fcntl.h>
  32 #include <ctype.h>
  33 #include <regex.h>
  34 
  35 #include <crm/crm.h>
  36 #include <crm/msg_xml.h>
  37 #include <crm/common/ipc.h>
  38 #include <crm/common/ipcs.h>
  39 #include <crm/cluster/internal.h>
  40 
  41 #include <crm/stonith-ng.h>
  42 #include <crm/fencing/internal.h>
  43 #include <crm/common/xml.h>
  44 
  45 #include <crm/common/util.h>
  46 #include <internal.h>
  47 
  48 #define TIMEOUT_MULTIPLY_FACTOR 1.2
  49 
  50 /* When one stonithd queries its peers for devices able to handle a fencing
  51  * request, each peer will reply with a list of such devices available to it.
  52  * Each reply will be parsed into a st_query_result_t, with each device's
  53  * information kept in a device_properties_t.
  54  */
  55 
  56 typedef struct device_properties_s {
  57     /* Whether access to this device has been verified */
  58     gboolean verified;
  59 
  60     /* The remaining members are indexed by the operation's "phase" */
  61 
  62     /* Whether this device has been executed in each phase */
  63     gboolean executed[st_phase_max];
  64     /* Whether this device is disallowed from executing in each phase */
  65     gboolean disallowed[st_phase_max];
  66     /* Action-specific timeout for each phase */
  67     int custom_action_timeout[st_phase_max];
  68     /* Action-specific maximum random delay for each phase */
  69     int delay_max[st_phase_max];
  70     /* Action-specific base delay for each phase */
  71     int delay_base[st_phase_max];
  72 } device_properties_t;
  73 
  74 typedef struct st_query_result_s {
  75     /* Name of peer that sent this result */
  76     char *host;
  77     /* Only try peers for non-topology based operations once */
  78     gboolean tried;
  79     /* Number of entries in the devices table */
  80     int ndevices;
  81     /* Devices available to this host that are capable of fencing the target */
  82     GHashTable *devices;
  83 } st_query_result_t;
  84 
  85 GHashTable *remote_op_list = NULL;
  86 void call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer);
  87 static void remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup);
  88 extern xmlNode *stonith_create_op(int call_id, const char *token, const char *op, xmlNode * data,
  89                                   int call_options);
  90 
  91 static void report_timeout_period(remote_fencing_op_t * op, int op_timeout);
  92 static int get_op_total_timeout(const remote_fencing_op_t *op,
  93                                 const st_query_result_t *chosen_peer);
  94 
  95 static gint
  96 sort_strings(gconstpointer a, gconstpointer b)
     /*  */
  97 {
  98     return strcmp(a, b);
  99 }
 100 
 101 static void
 102 free_remote_query(gpointer data)
     /*  */
 103 {
 104     if (data) {
 105         st_query_result_t *query = data;
 106 
 107         crm_trace("Free'ing query result from %s", query->host);
 108         g_hash_table_destroy(query->devices);
 109         free(query->host);
 110         free(query);
 111     }
 112 }
 113 
 114 struct peer_count_data {
 115     const remote_fencing_op_t *op;
 116     gboolean verified_only;
 117     int count;
 118 };
 119 
 120 /*!
 121  * \internal
 122  * \brief Increment a counter if a device has not been executed yet
 123  *
 124  * \param[in] key        Device ID (ignored)
 125  * \param[in] value      Device properties
 126  * \param[in] user_data  Peer count data
 127  */
 128 static void
 129 count_peer_device(gpointer key, gpointer value, gpointer user_data)
     /*  */
 130 {
 131     device_properties_t *props = (device_properties_t*)value;
 132     struct peer_count_data *data = user_data;
 133 
 134     if (!props->executed[data->op->phase]
 135         && (!data->verified_only || props->verified)) {
 136         ++(data->count);
 137     }
 138 }
 139 
 140 /*!
 141  * \internal
 142  * \brief Check the number of available devices in a peer's query results
 143  *
 144  * \param[in] op             Operation that results are for
 145  * \param[in] peer           Peer to count
 146  * \param[in] verified_only  Whether to count only verified devices
 147  *
 148  * \return Number of devices available to peer that were not already executed
 149  */
 150 static int
 151 count_peer_devices(const remote_fencing_op_t *op, const st_query_result_t *peer,
     /*  */
 152                    gboolean verified_only)
 153 {
 154     struct peer_count_data data;
 155 
 156     data.op = op;
 157     data.verified_only = verified_only;
 158     data.count = 0;
 159     if (peer) {
 160         g_hash_table_foreach(peer->devices, count_peer_device, &data);
 161     }
 162     return data.count;
 163 }
 164 
 165 /*!
 166  * \internal
 167  * \brief Search for a device in a query result
 168  *
 169  * \param[in] op      Operation that result is for
 170  * \param[in] peer    Query result for a peer
 171  * \param[in] device  Device ID to search for
 172  *
 173  * \return Device properties if found, NULL otherwise
 174  */
 175 static device_properties_t *
 176 find_peer_device(const remote_fencing_op_t *op, const st_query_result_t *peer,
     /*  */
 177                  const char *device)
 178 {
 179     device_properties_t *props = g_hash_table_lookup(peer->devices, device);
 180 
 181     return (props && !props->executed[op->phase]
 182            && !props->disallowed[op->phase])? props : NULL;
 183 }
 184 
 185 /*!
 186  * \internal
 187  * \brief Find a device in a peer's device list and mark it as executed
 188  *
 189  * \param[in]     op                     Operation that peer result is for
 190  * \param[in,out] peer                   Peer with results to search
 191  * \param[in]     device                 ID of device to mark as done
 192  * \param[in]     verified_devices_only  Only consider verified devices
 193  *
 194  * \return TRUE if device was found and marked, FALSE otherwise
 195  */
 196 static gboolean
 197 grab_peer_device(const remote_fencing_op_t *op, st_query_result_t *peer,
     /*  */
 198                  const char *device, gboolean verified_devices_only)
 199 {
 200     device_properties_t *props = find_peer_device(op, peer, device);
 201 
 202     if ((props == NULL) || (verified_devices_only && !props->verified)) {
 203         return FALSE;
 204     }
 205 
 206     crm_trace("Removing %s from %s (%d remaining)",
 207               device, peer->host, count_peer_devices(op, peer, FALSE));
 208     props->executed[op->phase] = TRUE;
 209     return TRUE;
 210 }
 211 
 212 static void
 213 clear_remote_op_timers(remote_fencing_op_t * op)
     /*  */
 214 {
 215     if (op->query_timer) {
 216         g_source_remove(op->query_timer);
 217         op->query_timer = 0;
 218     }
 219     if (op->op_timer_total) {
 220         g_source_remove(op->op_timer_total);
 221         op->op_timer_total = 0;
 222     }
 223     if (op->op_timer_one) {
 224         g_source_remove(op->op_timer_one);
 225         op->op_timer_one = 0;
 226     }
 227 }
 228 
 229 static void
 230 free_remote_op(gpointer data)
     /*  */
 231 {
 232     remote_fencing_op_t *op = data;
 233 
 234     crm_trace("Free'ing op %s for %s", op->id, op->target);
 235     crm_log_xml_debug(op->request, "Destroying");
 236 
 237     clear_remote_op_timers(op);
 238 
 239     free(op->id);
 240     free(op->action);
 241     free(op->target);
 242     free(op->client_id);
 243     free(op->client_name);
 244     free(op->originator);
 245 
 246     if (op->query_results) {
 247         g_list_free_full(op->query_results, free_remote_query);
 248     }
 249     if (op->request) {
 250         free_xml(op->request);
 251         op->request = NULL;
 252     }
 253     if (op->devices_list) {
 254         g_list_free_full(op->devices_list, free);
 255         op->devices_list = NULL;
 256     }
 257     g_list_free_full(op->automatic_list, free);
 258     free(op);
 259 }
 260 
 261 /*!
 262  * \internal
 263  * \brief Return an operation's originally requested action (before any remap)
 264  *
 265  * \param[in] op  Operation to check
 266  *
 267  * \return Operation's original action
 268  */
 269 static const char *
 270 op_requested_action(const remote_fencing_op_t *op)
     /*  */
 271 {
 272     return ((op->phase > st_phase_requested)? "reboot" : op->action);
 273 }
 274 
 275 /*!
 276  * \internal
 277  * \brief Remap a "reboot" operation to the "off" phase
 278  *
 279  * \param[in,out] op      Operation to remap
 280  */
 281 static void
 282 op_phase_off(remote_fencing_op_t *op)
     /*  */
 283 {
 284     crm_info("Remapping multiple-device reboot of %s (%s) to off",
 285              op->target, op->id);
 286     op->phase = st_phase_off;
 287 
 288     /* Happily, "off" and "on" are shorter than "reboot", so we can reuse the
 289      * memory allocation at each phase.
 290      */
 291     strcpy(op->action, "off");
 292 }
 293 
 294 /*!
 295  * \internal
 296  * \brief Advance a remapped reboot operation to the "on" phase
 297  *
 298  * \param[in,out] op  Operation to remap
 299  */
 300 static void
 301 op_phase_on(remote_fencing_op_t *op)
     /*  */
 302 {
 303     GListPtr iter = NULL;
 304 
 305     crm_info("Remapped off of %s complete, remapping to on for %s.%.8s",
 306              op->target, op->client_name, op->id);
 307     op->phase = st_phase_on;
 308     strcpy(op->action, "on");
 309 
 310     /* Skip devices with automatic unfencing, because the cluster will handle it
 311      * when the node rejoins.
 312      */
 313     for (iter = op->automatic_list; iter != NULL; iter = iter->next) {
 314         GListPtr match = g_list_find_custom(op->devices_list, iter->data,
 315                                             sort_strings);
 316 
 317         if (match) {
 318             op->devices_list = g_list_remove(op->devices_list, match->data);
 319         }
 320     }
 321     g_list_free_full(op->automatic_list, free);
 322     op->automatic_list = NULL;
 323 
 324     /* Rewind device list pointer */
 325     op->devices = op->devices_list;
 326 }
 327 
 328 /*!
 329  * \internal
 330  * \brief Reset a remapped reboot operation
 331  *
 332  * \param[in,out] op  Operation to reset
 333  */
 334 static void
 335 undo_op_remap(remote_fencing_op_t *op)
     /*  */
 336 {
 337     if (op->phase > 0) {
 338         crm_info("Undoing remap of reboot of %s for %s.%.8s",
 339                  op->target, op->client_name, op->id);
 340         op->phase = st_phase_requested;
 341         strcpy(op->action, "reboot");
 342     }
 343 }
 344 
 345 static xmlNode *
 346 create_op_done_notify(remote_fencing_op_t * op, int rc)
     /*  */
 347 {
 348     xmlNode *notify_data = create_xml_node(NULL, T_STONITH_NOTIFY_FENCE);
 349 
 350     crm_xml_add_int(notify_data, "state", op->state);
 351     crm_xml_add_int(notify_data, F_STONITH_RC, rc);
 352     crm_xml_add(notify_data, F_STONITH_TARGET, op->target);
 353     crm_xml_add(notify_data, F_STONITH_ACTION, op->action);
 354     crm_xml_add(notify_data, F_STONITH_DELEGATE, op->delegate);
 355     crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, op->id);
 356     crm_xml_add(notify_data, F_STONITH_ORIGIN, op->originator);
 357     crm_xml_add(notify_data, F_STONITH_CLIENTID, op->client_id);
 358     crm_xml_add(notify_data, F_STONITH_CLIENTNAME, op->client_name);
 359 
 360     return notify_data;
 361 }
 362 
 363 static void
 364 bcast_result_to_peers(remote_fencing_op_t * op, int rc)
     /*  */
 365 {
 366     static int count = 0;
 367     xmlNode *bcast = create_xml_node(NULL, T_STONITH_REPLY);
 368     xmlNode *notify_data = create_op_done_notify(op, rc);
 369 
 370     count++;
 371     crm_trace("Broadcasting result to peers");
 372     crm_xml_add(bcast, F_TYPE, T_STONITH_NOTIFY);
 373     crm_xml_add(bcast, F_SUBTYPE, "broadcast");
 374     crm_xml_add(bcast, F_STONITH_OPERATION, T_STONITH_NOTIFY);
 375     crm_xml_add_int(bcast, "count", count);
 376     add_message_xml(bcast, F_STONITH_CALLDATA, notify_data);
 377     send_cluster_message(NULL, crm_msg_stonith_ng, bcast, FALSE);
 378     free_xml(notify_data);
 379     free_xml(bcast);
 380 
 381     return;
 382 }
 383 
 384 static void
 385 handle_local_reply_and_notify(remote_fencing_op_t * op, xmlNode * data, int rc)
     /*  */
 386 {
 387     xmlNode *notify_data = NULL;
 388     xmlNode *reply = NULL;
 389 
 390     if (op->notify_sent == TRUE) {
 391         /* nothing to do */
 392         return;
 393     }
 394 
 395     /* Do notification with a clean data object */
 396     notify_data = create_op_done_notify(op, rc);
 397     crm_xml_add_int(data, "state", op->state);
 398     crm_xml_add(data, F_STONITH_TARGET, op->target);
 399     crm_xml_add(data, F_STONITH_OPERATION, op->action);
 400 
 401     reply = stonith_construct_reply(op->request, NULL, data, rc);
 402     crm_xml_add(reply, F_STONITH_DELEGATE, op->delegate);
 403 
 404     /* Send fencing OP reply to local client that initiated fencing */
 405     do_local_reply(reply, op->client_id, op->call_options & st_opt_sync_call, FALSE);
 406 
 407     /* bcast to all local clients that the fencing operation happend */
 408     do_stonith_notify(0, T_STONITH_NOTIFY_FENCE, rc, notify_data);
 409 
 410     /* mark this op as having notify's already sent */
 411     op->notify_sent = TRUE;
 412     free_xml(reply);
 413     free_xml(notify_data);
 414 }
 415 
 416 static void
 417 handle_duplicates(remote_fencing_op_t * op, xmlNode * data, int rc)
     /*  */
 418 {
 419     GListPtr iter = NULL;
 420 
 421     for (iter = op->duplicates; iter != NULL; iter = iter->next) {
 422         remote_fencing_op_t *other = iter->data;
 423 
 424         if (other->state == st_duplicate) {
 425             /* Ie. it hasn't timed out already */
 426             other->state = op->state;
 427             crm_debug("Peforming duplicate notification for %s@%s.%.8s = %s", other->client_name,
 428                       other->originator, other->id, pcmk_strerror(rc));
 429             remote_op_done(other, data, rc, TRUE);
 430 
 431         } else {
 432             crm_err("Skipping duplicate notification for %s@%s - %d", other->client_name,
 433                     other->originator, other->state);
 434         }
 435     }
 436 }
 437 
 438 /*!
 439  * \internal
 440  * \brief Finalize a remote operation.
 441  *
 442  * \description This function has two code paths.
 443  *
 444  * Path 1. This node is the owner of the operation and needs
 445  *         to notify the cpg group via a broadcast as to the operation's
 446  *         results.
 447  *
 448  * Path 2. The cpg broadcast is received. All nodes notify their local
 449  *         stonith clients the operation results.
 450  *
 451  * So, The owner of the operation first notifies the cluster of the result,
 452  * and once that cpg notify is received back it notifies all the local clients.
 453  *
 454  * Nodes that are passive watchers of the operation will receive the
 455  * broadcast and only need to notify their local clients the operation finished.
 456  *
 457  * \param op, The fencing operation to finalize
 458  * \param data, The xml msg reply (if present) of the last delegated fencing
 459  *              operation.
 460  * \param dup, Is this operation a duplicate, if so treat it a little differently
 461  *             making sure the broadcast is not sent out.
 462  */
 463 static void
 464 remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup)
     /*  */
 465 {
 466     int level = LOG_ERR;
 467     const char *subt = NULL;
 468     xmlNode *local_data = NULL;
 469 
 470     op->completed = time(NULL);
 471     clear_remote_op_timers(op);
 472     undo_op_remap(op);
 473 
 474     if (op->notify_sent == TRUE) {
 475         crm_err("Already sent notifications for '%s of %s by %s' (for=%s@%s.%.8s, state=%d): %s",
 476                 op->action, op->target, op->delegate ? op->delegate : "<no-one>",
 477                 op->client_name, op->originator, op->id, op->state, pcmk_strerror(rc));
 478         goto remote_op_done_cleanup;
 479     }
 480 
 481     if (!op->delegate && data && rc != -ENODEV && rc != -EHOSTUNREACH) {
 482         xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_TRACE);
 483         if(ndata) {
 484             op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE);
 485         } else { 
 486             op->delegate = crm_element_value_copy(data, F_ORIG);
 487         }
 488     }
 489 
 490     if (data == NULL) {
 491         data = create_xml_node(NULL, "remote-op");
 492         local_data = data;
 493     }
 494 
 495     /* Tell everyone the operation is done, we will continue
 496      * with doing the local notifications once we receive
 497      * the broadcast back. */
 498     subt = crm_element_value(data, F_SUBTYPE);
 499     if (dup == FALSE && safe_str_neq(subt, "broadcast")) {
 500         /* Defer notification until the bcast message arrives */
 501         bcast_result_to_peers(op, rc);
 502         goto remote_op_done_cleanup;
 503     }
 504 
 505     if (rc == pcmk_ok || dup) {
 506         level = LOG_NOTICE;
 507     } else if (safe_str_neq(op->originator, stonith_our_uname)) {
 508         level = LOG_NOTICE;
 509     }
 510 
 511     do_crm_log(level,
 512                "Operation %s of %s by %s for %s@%s.%.8s: %s",
 513                op->action, op->target, op->delegate ? op->delegate : "<no-one>",
 514                op->client_name, op->originator, op->id, pcmk_strerror(rc));
 515 
 516     handle_local_reply_and_notify(op, data, rc);
 517 
 518     if (dup == FALSE) {
 519         handle_duplicates(op, data, rc);
 520     }
 521 
 522     /* Free non-essential parts of the record
 523      * Keep the record around so we can query the history
 524      */
 525     if (op->query_results) {
 526         g_list_free_full(op->query_results, free_remote_query);
 527         op->query_results = NULL;
 528     }
 529 
 530     if (op->request) {
 531         free_xml(op->request);
 532         op->request = NULL;
 533     }
 534 
 535   remote_op_done_cleanup:
 536     free_xml(local_data);
 537 }
 538 
 539 static gboolean
 540 remote_op_watchdog_done(gpointer userdata)
     /*  */
 541 {
 542     remote_fencing_op_t *op = userdata;
 543 
 544     op->op_timer_one = 0;
 545 
 546     crm_notice("Self-fencing (%s) by %s for %s.%8s assumed complete",
 547                op->action, op->target, op->client_name, op->id);
 548     op->state = st_done;
 549     remote_op_done(op, NULL, pcmk_ok, FALSE);
 550     return FALSE;
 551 }
 552 
 553 static gboolean
 554 remote_op_timeout_one(gpointer userdata)
     /*  */
 555 {
 556     remote_fencing_op_t *op = userdata;
 557 
 558     op->op_timer_one = 0;
 559 
 560     crm_notice("Peer's fencing (%s) of %s for %s timed out" CRM_XS "id=%s",
 561                op->action, op->target, op->client_name, op->id);
 562     call_remote_stonith(op, NULL);
 563     return FALSE;
 564 }
 565 
 566 static gboolean
 567 remote_op_timeout(gpointer userdata)
     /*  */
 568 {
 569     remote_fencing_op_t *op = userdata;
 570 
 571     op->op_timer_total = 0;
 572 
 573     if (op->state == st_done) {
 574         crm_debug("Action %s (%s) for %s (%s) already completed",
 575                   op->action, op->id, op->target, op->client_name);
 576         return FALSE;
 577     }
 578 
 579     crm_debug("Action %s (%s) for %s (%s) timed out",
 580               op->action, op->id, op->target, op->client_name);
 581 
 582     if (op->phase == st_phase_on) {
 583         /* A remapped reboot operation timed out in the "on" phase, but the
 584          * "off" phase completed successfully, so quit trying any further
 585          * devices, and return success.
 586          */
 587         remote_op_done(op, NULL, pcmk_ok, FALSE);
 588         return FALSE;
 589     }
 590 
 591     op->state = st_failed;
 592 
 593     remote_op_done(op, NULL, -ETIME, FALSE);
 594 
 595     return FALSE;
 596 }
 597 
 598 static gboolean
 599 remote_op_query_timeout(gpointer data)
     /*  */
 600 {
 601     remote_fencing_op_t *op = data;
 602 
 603     op->query_timer = 0;
 604     if (op->state == st_done) {
 605         crm_debug("Operation %s for %s already completed", op->id, op->target);
 606     } else if (op->state == st_exec) {
 607         crm_debug("Operation %s for %s already in progress", op->id, op->target);
 608     } else if (op->query_results) {
 609         crm_debug("Query %s for %s complete: %d", op->id, op->target, op->state);
 610         call_remote_stonith(op, NULL);
 611     } else {
 612         crm_debug("Query %s for %s timed out: %d", op->id, op->target, op->state);
 613         if (op->op_timer_total) {
 614             g_source_remove(op->op_timer_total);
 615             op->op_timer_total = 0;
 616         }
 617         remote_op_timeout(op);
 618     }
 619 
 620     return FALSE;
 621 }
 622 
 623 static gboolean
 624 topology_is_empty(stonith_topology_t *tp)
     /*  */
 625 {
 626     int i;
 627 
 628     if (tp == NULL) {
 629         return TRUE;
 630     }
 631 
 632     for (i = 0; i < ST_LEVEL_MAX; i++) {
 633         if (tp->levels[i] != NULL) {
 634             return FALSE;
 635         }
 636     }
 637     return TRUE;
 638 }
 639 
 640 /*!
 641  * \internal
 642  * \brief Add a device to an operation's automatic unfencing list
 643  *
 644  * \param[in,out] op      Operation to modify
 645  * \param[in]     device  Device ID to add
 646  */
 647 static void
 648 add_required_device(remote_fencing_op_t *op, const char *device)
     /*  */
 649 {
 650     GListPtr match  = g_list_find_custom(op->automatic_list, device,
 651                                          sort_strings);
 652 
 653     if (!match) {
 654         op->automatic_list = g_list_prepend(op->automatic_list, strdup(device));
 655     }
 656 }
 657 
 658 /*!
 659  * \internal
 660  * \brief Remove a device from the automatic unfencing list
 661  *
 662  * \param[in,out] op      Operation to modify
 663  * \param[in]     device  Device ID to remove
 664  */
 665 static void
 666 remove_required_device(remote_fencing_op_t *op, const char *device)
     /*  */
 667 {
 668     GListPtr match = g_list_find_custom(op->automatic_list, device,
 669                                         sort_strings);
 670 
 671     if (match) {
 672         op->automatic_list = g_list_remove(op->automatic_list, match->data);
 673     }
 674 }
 675 
 676 /* deep copy the device list */
 677 static void
 678 set_op_device_list(remote_fencing_op_t * op, GListPtr devices)
     /*  */
 679 {
 680     GListPtr lpc = NULL;
 681 
 682     if (op->devices_list) {
 683         g_list_free_full(op->devices_list, free);
 684         op->devices_list = NULL;
 685     }
 686     for (lpc = devices; lpc != NULL; lpc = lpc->next) {
 687         op->devices_list = g_list_append(op->devices_list, strdup(lpc->data));
 688     }
 689     op->devices = op->devices_list;
 690 }
 691 
 692 /*!
 693  * \internal
 694  * \brief Check whether a node matches a topology target
 695  *
 696  * \param[in] tp    Topology table entry to check
 697  * \param[in] node  Name of node to check
 698  *
 699  * \return TRUE if node matches topology target
 700  */
 701 static gboolean
 702 topology_matches(const stonith_topology_t *tp, const char *node)
     /*  */
 703 {
 704     regex_t r_patt;
 705 
 706     CRM_CHECK(node && tp && tp->target, return FALSE);
 707     switch(tp->kind) {
 708         case 2:
 709             /* This level targets by attribute, so tp->target is a NAME=VALUE pair
 710              * of a permanent attribute applied to targeted nodes. The test below
 711              * relies on the locally cached copy of the CIB, so if fencing needs to
 712              * be done before the initial CIB is received or after a malformed CIB
 713              * is received, then the topology will be unable to be used.
 714              */
 715             if (node_has_attr(node, tp->target_attribute, tp->target_value)) {
 716                 crm_notice("Matched %s with %s by attribute", node, tp->target);
 717                 return TRUE;
 718             }
 719             break;
 720         case 1:
 721             /* This level targets by name, so tp->target is a regular expression
 722              * matching names of nodes to be targeted.
 723              */
 724 
 725             if (regcomp(&r_patt, tp->target_pattern, REG_EXTENDED|REG_NOSUB)) {
 726                 crm_info("Bad regex '%s' for fencing level", tp->target);
 727             } else {
 728                 int status = regexec(&r_patt, node, 0, NULL, 0);
 729 
 730                 regfree(&r_patt);
 731                 if (status == 0) {
 732                     crm_notice("Matched %s with %s by name", node, tp->target);
 733                     return TRUE;
 734                 }
 735             }
 736             break;
 737         case 0:
 738             crm_trace("Testing %s against %s", node, tp->target);
 739             return safe_str_eq(tp->target, node);
 740     }
 741     crm_trace("No match for %s with %s", node, tp->target);
 742     return FALSE;
 743 }
 744 
 745 stonith_topology_t *
 746 find_topology_for_host(const char *host) 
     /*  */
 747 {
 748     GHashTableIter tIter;
 749     stonith_topology_t *tp = g_hash_table_lookup(topology, host);
 750 
 751     if(tp != NULL) {
 752         crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
 753         return tp;
 754     }
 755 
 756     g_hash_table_iter_init(&tIter, topology);
 757     while (g_hash_table_iter_next(&tIter, NULL, (gpointer *) & tp)) {
 758         if (topology_matches(tp, host)) {
 759             crm_trace("Found %s for %s in %d entries", tp->target, host, g_hash_table_size(topology));
 760             return tp;
 761         }
 762     }
 763 
 764     crm_trace("No matches for %s in %d topology entries", host, g_hash_table_size(topology));
 765     return NULL;
 766 }
 767 
 768 /*!
 769  * \internal
 770  * \brief Set fencing operation's device list to target's next topology level
 771  *
 772  * \param[in,out] op  Remote fencing operation to modify
 773  *
 774  * \return pcmk_ok if successful, target was not specified (i.e. queries) or
 775  *         target has no topology, or -EINVAL if no more topology levels to try
 776  */
 777 static int
 778 stonith_topology_next(remote_fencing_op_t * op)
     /*  */
 779 {
 780     stonith_topology_t *tp = NULL;
 781 
 782     if (op->target) {
 783         /* Queries don't have a target set */
 784         tp = find_topology_for_host(op->target);
 785     }
 786     if (topology_is_empty(tp)) {
 787         return pcmk_ok;
 788     }
 789 
 790     set_bit(op->call_options, st_opt_topology);
 791 
 792     /* This is a new level, so undo any remapping left over from previous */
 793     undo_op_remap(op);
 794 
 795     do {
 796         op->level++;
 797 
 798     } while (op->level < ST_LEVEL_MAX && tp->levels[op->level] == NULL);
 799 
 800     if (op->level < ST_LEVEL_MAX) {
 801         crm_trace("Attempting fencing level %d for %s (%d devices) - %s@%s.%.8s",
 802                   op->level, op->target, g_list_length(tp->levels[op->level]),
 803                   op->client_name, op->originator, op->id);
 804         set_op_device_list(op, tp->levels[op->level]);
 805 
 806         if (g_list_next(op->devices_list) && safe_str_eq(op->action, "reboot")) {
 807             /* A reboot has been requested for a topology level with multiple
 808              * devices. Instead of rebooting the devices sequentially, we will
 809              * turn them all off, then turn them all on again. (Think about
 810              * switched power outlets for redundant power supplies.)
 811              */
 812             op_phase_off(op);
 813         }
 814         return pcmk_ok;
 815     }
 816 
 817     crm_notice("All fencing options to fence %s for %s@%s.%.8s failed",
 818                op->target, op->client_name, op->originator, op->id);
 819     return -EINVAL;
 820 }
 821 
 822 /*!
 823  * \brief Check to see if this operation is a duplicate of another in flight
 824  * operation. If so merge this operation into the inflight operation, and mark
 825  * it as a duplicate.
 826  */
 827 static void
 828 merge_duplicates(remote_fencing_op_t * op)
     /*  */
 829 {
 830     GHashTableIter iter;
 831     remote_fencing_op_t *other = NULL;
 832 
 833     time_t now = time(NULL);
 834 
 835     g_hash_table_iter_init(&iter, remote_op_list);
 836     while (g_hash_table_iter_next(&iter, NULL, (void **)&other)) {
 837         crm_node_t *peer = NULL;
 838         const char *other_action = op_requested_action(other);
 839 
 840         if (other->state > st_exec) {
 841             /* Must be in-progress */
 842             continue;
 843         } else if (safe_str_neq(op->target, other->target)) {
 844             /* Must be for the same node */
 845             continue;
 846         } else if (safe_str_neq(op->action, other_action)) {
 847             crm_trace("Must be for the same action: %s vs. %s",
 848                       op->action, other_action);
 849             continue;
 850         } else if (safe_str_eq(op->client_name, other->client_name)) {
 851             crm_trace("Must be for different clients: %s", op->client_name);
 852             continue;
 853         } else if (safe_str_eq(other->target, other->originator)) {
 854             crm_trace("Can't be a suicide operation: %s", other->target);
 855             continue;
 856         }
 857 
 858         peer = crm_get_peer(0, other->originator);
 859         if(fencing_peer_active(peer) == FALSE) {
 860             crm_notice("Failing stonith action %s for node %s originating from %s@%s.%.8s: Originator is dead",
 861                        other->action, other->target, other->client_name, other->originator, other->id);
 862             other->state = st_failed;
 863             continue;
 864 
 865         } else if(other->total_timeout > 0 && now > (other->total_timeout + other->created)) {
 866             crm_info("Stonith action %s for node %s originating from %s@%s.%.8s is too old: %d vs. %d + %d",
 867                      other->action, other->target, other->client_name, other->originator, other->id,
 868                      now, other->created, other->total_timeout);
 869             continue;
 870         }
 871 
 872         /* There is another in-flight request to fence the same host
 873          * Piggyback on that instead.  If it fails, so do we.
 874          */
 875         other->duplicates = g_list_append(other->duplicates, op);
 876         if (other->total_timeout == 0) {
 877             crm_trace("Making a best-guess as to the timeout used");
 878             other->total_timeout = op->total_timeout =
 879                 TIMEOUT_MULTIPLY_FACTOR * get_op_total_timeout(op, NULL);
 880         }
 881         crm_notice
 882             ("Merging stonith action %s for node %s originating from client %s.%.8s with identical request from %s@%s.%.8s (%ds)",
 883              op->action, op->target, op->client_name, op->id, other->client_name, other->originator,
 884              other->id, other->total_timeout);
 885         report_timeout_period(op, other->total_timeout);
 886         op->state = st_duplicate;
 887     }
 888 }
 889 
 890 static uint32_t fencing_active_peers(void)
     /*  */
 891 {
 892     uint32_t count = 0;
 893     crm_node_t *entry;
 894     GHashTableIter gIter;
 895 
 896     g_hash_table_iter_init(&gIter, crm_peer_cache);
 897     while (g_hash_table_iter_next(&gIter, NULL, (void **)&entry)) {
 898         if(fencing_peer_active(entry)) {
 899             count++;
 900         }
 901     }
 902     return count;
 903 }
 904 
 905 int
 906 stonith_manual_ack(xmlNode * msg, remote_fencing_op_t * op)
     /*  */
 907 {
 908     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_ERR);
 909 
 910     op->state = st_done;
 911     op->completed = time(NULL);
 912     op->delegate = strdup("a human");
 913 
 914     crm_notice("Injecting manual confirmation that %s is safely off/down",
 915                crm_element_value(dev, F_STONITH_TARGET));
 916 
 917     remote_op_done(op, msg, pcmk_ok, FALSE);
 918 
 919     /* Replies are sent via done_cb->stonith_send_async_reply()->do_local_reply() */
 920     return -EINPROGRESS;
 921 }
 922 
 923 char *
 924 stonith_get_peer_name(unsigned int nodeid)
     /*  */
 925 {
 926     crm_node_t *node = crm_find_peer(nodeid, NULL);
 927     char *nodename = NULL;
 928 
 929     if (node && node->uname) {
 930         return strdup(node->uname);
 931 
 932     } else if ((nodename = get_node_name(nodeid))) {
 933         return nodename;
 934 
 935     } else {
 936         const char *last_known_name = g_hash_table_lookup(known_peer_names, GUINT_TO_POINTER(nodeid));
 937 
 938         if (last_known_name) {
 939             crm_debug("Use the last known name %s for nodeid %u", last_known_name, nodeid);
 940             return strdup(last_known_name);
 941         }
 942     }
 943 
 944     return NULL;
 945 }
 946 
 947 /*!
 948  * \internal
 949  * \brief Create a new remote stonith operation
 950  *
 951  * \param[in] client   ID of local stonith client that initiated the operation
 952  * \param[in] request  The request from the client that started the operation
 953  * \param[in] peer     TRUE if this operation is owned by another stonith peer
 954  *                     (an operation owned by one peer is stored on all peers,
 955  *                     but only the owner executes it; all nodes get the results
 956  *                     once the owner finishes execution)
 957  */
 958 void *
 959 create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer)
     /*  */
 960 {
 961     remote_fencing_op_t *op = NULL;
 962     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE);
 963     int call_options = 0;
 964 
 965     if (remote_op_list == NULL) {
 966         remote_op_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_remote_op);
 967     }
 968 
 969     /* If this operation is owned by another node, check to make
 970      * sure we haven't already created this operation. */
 971     if (peer && dev) {
 972         const char *op_id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
 973 
 974         CRM_CHECK(op_id != NULL, return NULL);
 975 
 976         op = g_hash_table_lookup(remote_op_list, op_id);
 977         if (op) {
 978             crm_debug("%s already exists", op_id);
 979             return op;
 980         }
 981     }
 982 
 983     op = calloc(1, sizeof(remote_fencing_op_t));
 984 
 985     crm_element_value_int(request, F_STONITH_TIMEOUT, &(op->base_timeout));
 986 
 987     if (peer && dev) {
 988         op->id = crm_element_value_copy(dev, F_STONITH_REMOTE_OP_ID);
 989     } else {
 990         op->id = crm_generate_uuid();
 991     }
 992 
 993     g_hash_table_replace(remote_op_list, op->id, op);
 994     CRM_LOG_ASSERT(g_hash_table_lookup(remote_op_list, op->id) != NULL);
 995     crm_trace("Created %s", op->id);
 996 
 997     op->state = st_query;
 998     op->replies_expected = fencing_active_peers();
 999     op->action = crm_element_value_copy(dev, F_STONITH_ACTION);
1000     op->originator = crm_element_value_copy(dev, F_STONITH_ORIGIN);
1001     op->delegate = crm_element_value_copy(dev, F_STONITH_DELEGATE); /* May not be set */
1002     op->created = time(NULL);
1003 
1004     if (op->originator == NULL) {
1005         /* Local or relayed request */
1006         op->originator = strdup(stonith_our_uname);
1007     }
1008 
1009     CRM_LOG_ASSERT(client != NULL);
1010     if (client) {
1011         op->client_id = strdup(client);
1012     }
1013 
1014     op->client_name = crm_element_value_copy(request, F_STONITH_CLIENTNAME);
1015 
1016     op->target = crm_element_value_copy(dev, F_STONITH_TARGET);
1017     op->request = copy_xml(request);    /* TODO: Figure out how to avoid this */
1018     crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options);
1019     op->call_options = call_options;
1020 
1021     crm_element_value_int(request, F_STONITH_CALLID, &(op->client_callid));
1022 
1023     crm_trace("%s new stonith op: %s - %s of %s for %s",
1024               (peer
1025                && dev) ? "Recorded" : "Generated", op->id, op->action, op->target, op->client_name);
1026 
1027     if (op->call_options & st_opt_cs_nodeid) {
1028         int nodeid = crm_atoi(op->target, NULL);
1029         char *nodename = stonith_get_peer_name(nodeid);
1030 
1031         /* Ensure the conversion only happens once */
1032         op->call_options &= ~st_opt_cs_nodeid;
1033 
1034         if (nodename) {
1035             free(op->target);
1036             op->target = nodename;
1037 
1038         } else {
1039             crm_warn("Could not expand nodeid '%s' into a host name", op->target);
1040         }
1041     }
1042 
1043     /* check to see if this is a duplicate operation of another in-flight operation */
1044     merge_duplicates(op);
1045 
1046     return op;
1047 }
1048 
1049 remote_fencing_op_t *
1050 initiate_remote_stonith_op(crm_client_t * client, xmlNode * request, gboolean manual_ack)
     /*  */
1051 {
1052     int query_timeout = 0;
1053     xmlNode *query = NULL;
1054     const char *client_id = NULL;
1055     remote_fencing_op_t *op = NULL;
1056 
1057     if (client) {
1058         client_id = client->id;
1059     } else {
1060         client_id = crm_element_value(request, F_STONITH_CLIENTID);
1061     }
1062 
1063     CRM_LOG_ASSERT(client_id != NULL);
1064     op = create_remote_stonith_op(client_id, request, FALSE);
1065     op->owner = TRUE;
1066     if (manual_ack) {
1067         crm_notice("Initiating manual confirmation for %s: %s",
1068                    op->target, op->id);
1069         return op;
1070     }
1071 
1072     CRM_CHECK(op->action, return NULL);
1073 
1074     if (stonith_topology_next(op) != pcmk_ok) {
1075         op->state = st_failed;
1076     }
1077 
1078     switch (op->state) {
1079         case st_failed:
1080             crm_warn("Could not request peer fencing (%s) of %s "
1081                      CRM_XS " id=%s", op->action, op->target, op->id);
1082             remote_op_done(op, NULL, -EINVAL, FALSE);
1083             return op;
1084 
1085         case st_duplicate:
1086             crm_info("Requesting peer fencing (%s) of %s (duplicate) "
1087                      CRM_XS " id=%s", op->action, op->target, op->id);
1088             return op;
1089 
1090         default:
1091             crm_notice("Requesting peer fencing (%s) of %s "
1092                        CRM_XS " id=%s state=%d",
1093                        op->action, op->target, op->id, op->state);
1094     }
1095 
1096     query = stonith_create_op(op->client_callid, op->id, STONITH_OP_QUERY,
1097                               NULL, op->call_options);
1098 
1099     crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id);
1100     crm_xml_add(query, F_STONITH_TARGET, op->target);
1101     crm_xml_add(query, F_STONITH_ACTION, op_requested_action(op));
1102     crm_xml_add(query, F_STONITH_ORIGIN, op->originator);
1103     crm_xml_add(query, F_STONITH_CLIENTID, op->client_id);
1104     crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name);
1105     crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout);
1106 
1107     send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE);
1108     free_xml(query);
1109 
1110     query_timeout = op->base_timeout * TIMEOUT_MULTIPLY_FACTOR;
1111     op->query_timer = g_timeout_add((1000 * query_timeout), remote_op_query_timeout, op);
1112 
1113     return op;
1114 }
1115 
1116 enum find_best_peer_options {
1117     /*! Skip checking the target peer for capable fencing devices */
1118     FIND_PEER_SKIP_TARGET = 0x0001,
1119     /*! Only check the target peer for capable fencing devices */
1120     FIND_PEER_TARGET_ONLY = 0x0002,
1121     /*! Skip peers and devices that are not verified */
1122     FIND_PEER_VERIFIED_ONLY = 0x0004,
1123 };
1124 
1125 static st_query_result_t *
1126 find_best_peer(const char *device, remote_fencing_op_t * op, enum find_best_peer_options options)
     /*  */
1127 {
1128     GListPtr iter = NULL;
1129     gboolean verified_devices_only = (options & FIND_PEER_VERIFIED_ONLY) ? TRUE : FALSE;
1130 
1131     if (!device && is_set(op->call_options, st_opt_topology)) {
1132         return NULL;
1133     }
1134 
1135     for (iter = op->query_results; iter != NULL; iter = iter->next) {
1136         st_query_result_t *peer = iter->data;
1137 
1138         crm_trace("Testing result from %s for %s with %d devices: %d %x",
1139                   peer->host, op->target, peer->ndevices, peer->tried, options);
1140         if ((options & FIND_PEER_SKIP_TARGET) && safe_str_eq(peer->host, op->target)) {
1141             continue;
1142         }
1143         if ((options & FIND_PEER_TARGET_ONLY) && safe_str_neq(peer->host, op->target)) {
1144             continue;
1145         }
1146 
1147         if (is_set(op->call_options, st_opt_topology)) {
1148 
1149             if (grab_peer_device(op, peer, device, verified_devices_only)) {
1150                 return peer;
1151             }
1152 
1153         } else if ((peer->tried == FALSE)
1154                    && count_peer_devices(op, peer, verified_devices_only)) {
1155 
1156             /* No topology: Use the current best peer */
1157             crm_trace("Simple fencing");
1158             return peer;
1159         }
1160     }
1161 
1162     return NULL;
1163 }
1164 
1165 static st_query_result_t *
1166 stonith_choose_peer(remote_fencing_op_t * op)
     /*  */
1167 {
1168     const char *device = NULL;
1169     st_query_result_t *peer = NULL;
1170     uint32_t active = fencing_active_peers();
1171 
1172     do {
1173         if (op->devices) {
1174             device = op->devices->data;
1175             crm_trace("Checking for someone to fence (%s) %s with %s",
1176                       op->action, op->target, device);
1177         } else {
1178             crm_trace("Checking for someone to fence (%s) %s",
1179                       op->action, op->target);
1180         }
1181 
1182         /* Best choice is a peer other than the target with verified access */
1183         peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET|FIND_PEER_VERIFIED_ONLY);
1184         if (peer) {
1185             crm_trace("Found verified peer %s for %s", peer->host, device?device:"<any>");
1186             return peer;
1187         }
1188 
1189         if(op->query_timer != 0 && op->replies < QB_MIN(op->replies_expected, active)) {
1190             crm_trace("Waiting before looking for unverified devices to fence %s", op->target);
1191             return NULL;
1192         }
1193 
1194         /* If no other peer has verified access, next best is unverified access */
1195         peer = find_best_peer(device, op, FIND_PEER_SKIP_TARGET);
1196         if (peer) {
1197             crm_trace("Found best unverified peer %s", peer->host);
1198             return peer;
1199         }
1200 
1201         /* If no other peer can do it, last option is self-fencing
1202          * (which is never allowed for the "on" phase of a remapped reboot)
1203          */
1204         if (op->phase != st_phase_on) {
1205             peer = find_best_peer(device, op, FIND_PEER_TARGET_ONLY);
1206             if (peer) {
1207                 crm_trace("%s will fence itself", peer->host);
1208                 return peer;
1209             }
1210         }
1211 
1212         /* Try the next fencing level if there is one (unless we're in the "on"
1213          * phase of a remapped "reboot", because we ignore errors in that case)
1214          */
1215     } while ((op->phase != st_phase_on)
1216              && is_set(op->call_options, st_opt_topology)
1217              && stonith_topology_next(op) == pcmk_ok);
1218 
1219     crm_notice("Couldn't find anyone to fence (%s) %s with %s",
1220                op->action, op->target, (device? device : "any device"));
1221     return NULL;
1222 }
1223 
1224 static int
1225 get_device_timeout(const remote_fencing_op_t *op, const st_query_result_t *peer,
     /*  */
1226                    const char *device)
1227 {
1228     device_properties_t *props;
1229 
1230     if (!peer || !device) {
1231         return op->base_timeout;
1232     }
1233 
1234     props = g_hash_table_lookup(peer->devices, device);
1235     if (!props) {
1236         return op->base_timeout;
1237     }
1238 
1239     return (props->custom_action_timeout[op->phase]?
1240            props->custom_action_timeout[op->phase] : op->base_timeout)
1241            + props->delay_max[op->phase];
1242 }
1243 
1244 struct timeout_data {
1245     const remote_fencing_op_t *op;
1246     const st_query_result_t *peer;
1247     int total_timeout;
1248 };
1249 
1250 /*!
1251  * \internal
1252  * \brief Add timeout to a total if device has not been executed yet
1253  *
1254  * \param[in] key        GHashTable key (device ID)
1255  * \param[in] value      GHashTable value (device properties)
1256  * \param[in] user_data  Timeout data
1257  */
1258 static void
1259 add_device_timeout(gpointer key, gpointer value, gpointer user_data)
     /*  */
1260 {
1261     const char *device_id = key;
1262     device_properties_t *props = value;
1263     struct timeout_data *timeout = user_data;
1264 
1265     if (!props->executed[timeout->op->phase]
1266         && !props->disallowed[timeout->op->phase]) {
1267         timeout->total_timeout += get_device_timeout(timeout->op,
1268                                                      timeout->peer, device_id);
1269     }
1270 }
1271 
1272 static int
1273 get_peer_timeout(const remote_fencing_op_t *op, const st_query_result_t *peer)
     /*  */
1274 {
1275     struct timeout_data timeout;
1276 
1277     timeout.op = op;
1278     timeout.peer = peer;
1279     timeout.total_timeout = 0;
1280 
1281     g_hash_table_foreach(peer->devices, add_device_timeout, &timeout);
1282 
1283     return (timeout.total_timeout? timeout.total_timeout : op->base_timeout);
1284 }
1285 
1286 static int
1287 get_op_total_timeout(const remote_fencing_op_t *op,
     /*  */
1288                      const st_query_result_t *chosen_peer)
1289 {
1290     int total_timeout = 0;
1291     stonith_topology_t *tp = find_topology_for_host(op->target);
1292 
1293     if (is_set(op->call_options, st_opt_topology) && tp) {
1294         int i;
1295         GListPtr device_list = NULL;
1296         GListPtr iter = NULL;
1297 
1298         /* Yep, this looks scary, nested loops all over the place.
1299          * Here is what is going on.
1300          * Loop1: Iterate through fencing levels.
1301          * Loop2: If a fencing level has devices, loop through each device
1302          * Loop3: For each device in a fencing level, see what peer owns it
1303          *        and what that peer has reported the timeout is for the device.
1304          */
1305         for (i = 0; i < ST_LEVEL_MAX; i++) {
1306             if (!tp->levels[i]) {
1307                 continue;
1308             }
1309             for (device_list = tp->levels[i]; device_list; device_list = device_list->next) {
1310                 for (iter = op->query_results; iter != NULL; iter = iter->next) {
1311                     const st_query_result_t *peer = iter->data;
1312 
1313                     if (find_peer_device(op, peer, device_list->data)) {
1314                         total_timeout += get_device_timeout(op, peer,
1315                                                             device_list->data);
1316                         break;
1317                     }
1318                 }               /* End Loop3: match device with peer that owns device, find device's timeout period */
1319             }                   /* End Loop2: iterate through devices at a specific level */
1320         }                       /*End Loop1: iterate through fencing levels */
1321 
1322     } else if (chosen_peer) {
1323         total_timeout = get_peer_timeout(op, chosen_peer);
1324     } else {
1325         total_timeout = op->base_timeout;
1326     }
1327 
1328     return total_timeout ? total_timeout : op->base_timeout;
1329 }
1330 
1331 static void
1332 report_timeout_period(remote_fencing_op_t * op, int op_timeout)
     /*  */
1333 {
1334     GListPtr iter = NULL;
1335     xmlNode *update = NULL;
1336     const char *client_node = NULL;
1337     const char *client_id = NULL;
1338     const char *call_id = NULL;
1339 
1340     if (op->call_options & st_opt_sync_call) {
1341         /* There is no reason to report the timeout for a synchronous call. It
1342          * is impossible to use the reported timeout to do anything when the client
1343          * is blocking for the response.  This update is only important for
1344          * async calls that require a callback to report the results in. */
1345         return;
1346     } else if (!op->request) {
1347         return;
1348     }
1349 
1350     crm_trace("Reporting timeout for %s.%.8s", op->client_name, op->id);
1351     client_node = crm_element_value(op->request, F_STONITH_CLIENTNODE);
1352     call_id = crm_element_value(op->request, F_STONITH_CALLID);
1353     client_id = crm_element_value(op->request, F_STONITH_CLIENTID);
1354     if (!client_node || !call_id || !client_id) {
1355         return;
1356     }
1357 
1358     if (safe_str_eq(client_node, stonith_our_uname)) {
1359         /* The client is connected to this node, send the update direclty to them */
1360         do_stonith_async_timeout_update(client_id, call_id, op_timeout);
1361         return;
1362     }
1363 
1364     /* The client is connected to another node, relay this update to them */
1365     update = stonith_create_op(op->client_callid, op->id, STONITH_OP_TIMEOUT_UPDATE, NULL, 0);
1366     crm_xml_add(update, F_STONITH_REMOTE_OP_ID, op->id);
1367     crm_xml_add(update, F_STONITH_CLIENTID, client_id);
1368     crm_xml_add(update, F_STONITH_CALLID, call_id);
1369     crm_xml_add_int(update, F_STONITH_TIMEOUT, op_timeout);
1370 
1371     send_cluster_message(crm_get_peer(0, client_node), crm_msg_stonith_ng, update, FALSE);
1372 
1373     free_xml(update);
1374 
1375     for (iter = op->duplicates; iter != NULL; iter = iter->next) {
1376         remote_fencing_op_t *dup = iter->data;
1377 
1378         crm_trace("Reporting timeout for duplicate %s.%.8s", dup->client_name, dup->id);
1379         report_timeout_period(iter->data, op_timeout);
1380     }
1381 }
1382 
1383 /*!
1384  * \internal
1385  * \brief Advance an operation to the next device in its topology
1386  *
1387  * \param[in,out] op      Operation to advance
1388  * \param[in]     device  ID of device just completed
1389  * \param[in]     msg     XML reply that contained device result (if available)
1390  * \param[in]     rc      Return code of device's execution
1391  */
1392 static void
1393 advance_op_topology(remote_fencing_op_t *op, const char *device, xmlNode *msg,
     /*  */
1394                     int rc)
1395 {
1396     /* Advance to the next device at this topology level, if any */
1397     if (op->devices) {
1398         op->devices = op->devices->next;
1399     }
1400 
1401     /* Handle automatic unfencing if an "on" action was requested */
1402     if ((op->phase == st_phase_requested) && safe_str_eq(op->action, "on")) {
1403         /* If the device we just executed was required, it's not anymore */
1404         remove_required_device(op, device);
1405 
1406         /* If there are no more devices at this topology level, run through any
1407          * remaining devices with automatic unfencing
1408          */
1409         if (op->devices == NULL) {
1410             op->devices = op->automatic_list;
1411         }
1412     }
1413 
1414     if ((op->devices == NULL) && (op->phase == st_phase_off)) {
1415         /* We're done with this level and with required devices, but we had
1416          * remapped "reboot" to "off", so start over with "on". If any devices
1417          * need to be turned back on, op->devices will be non-NULL after this.
1418          */
1419         op_phase_on(op);
1420     }
1421 
1422     if (op->devices) {
1423         /* Necessary devices remain, so execute the next one */
1424         crm_trace("Next for %s on behalf of %s@%s (rc was %d)",
1425                   op->target, op->originator, op->client_name, rc);
1426         call_remote_stonith(op, NULL);
1427     } else {
1428         /* We're done with all devices and phases, so finalize operation */
1429         crm_trace("Marking complex fencing op for %s as complete", op->target);
1430         op->state = st_done;
1431         remote_op_done(op, msg, rc, FALSE);
1432     }
1433 }
1434 
1435 void
1436 call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer)
     /*  */
1437 {
1438     const char *device = NULL;
1439     int timeout = op->base_timeout;
1440 
1441     crm_trace("State for %s.%.8s: %s %d", op->target, op->client_name, op->id, op->state);
1442     if (peer == NULL && !is_set(op->call_options, st_opt_topology)) {
1443         peer = stonith_choose_peer(op);
1444     }
1445 
1446     if (!op->op_timer_total) {
1447         int total_timeout = get_op_total_timeout(op, peer);
1448 
1449         op->total_timeout = TIMEOUT_MULTIPLY_FACTOR * total_timeout;
1450         op->op_timer_total = g_timeout_add(1000 * op->total_timeout, remote_op_timeout, op);
1451         report_timeout_period(op, op->total_timeout);
1452         crm_info("Total timeout set to %d for peer's fencing of %s for %s"
1453                  CRM_XS "id=%s",
1454                  total_timeout, op->target, op->client_name, op->id);
1455     }
1456 
1457     if (is_set(op->call_options, st_opt_topology) && op->devices) {
1458         /* Ignore any peer preference, they might not have the device we need */
1459         /* When using topology, stonith_choose_peer() removes the device from
1460          * further consideration, so be sure to calculate timeout beforehand */
1461         peer = stonith_choose_peer(op);
1462 
1463         device = op->devices->data;
1464         timeout = get_device_timeout(op, peer, device);
1465     }
1466 
1467     if (peer) {
1468         int timeout_one = 0;
1469         xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0);
1470 
1471         crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id);
1472         crm_xml_add(remote_op, F_STONITH_TARGET, op->target);
1473         crm_xml_add(remote_op, F_STONITH_ACTION, op->action);
1474         crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator);
1475         crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id);
1476         crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name);
1477         crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout);
1478         crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options);
1479 
1480         if (device) {
1481             timeout_one = TIMEOUT_MULTIPLY_FACTOR *
1482                           get_device_timeout(op, peer, device);
1483             crm_info("Requesting that '%s' perform op '%s %s' with '%s' for %s (%ds)", peer->host,
1484                      op->target, op->action, device, op->client_name, timeout_one);
1485             crm_xml_add(remote_op, F_STONITH_DEVICE, device);
1486             crm_xml_add(remote_op, F_STONITH_MODE, "slave");
1487 
1488         } else {
1489             timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(op, peer);
1490             crm_info("Requesting that '%s' perform op '%s %s' for %s (%ds, %ds)",
1491                      peer->host, op->target, op->action, op->client_name, timeout_one, stonith_watchdog_timeout_ms);
1492             crm_xml_add(remote_op, F_STONITH_MODE, "smart");
1493 
1494         }
1495 
1496         op->state = st_exec;
1497         if (op->op_timer_one) {
1498             g_source_remove(op->op_timer_one);
1499         }
1500 
1501         if(stonith_watchdog_timeout_ms > 0 && device && safe_str_eq(device, "watchdog")) {
1502             crm_notice("Waiting %ds for %s to self-fence (%s) for %s.%.8s (%p)",
1503                        stonith_watchdog_timeout_ms/1000, op->target,
1504                        op->action, op->client_name, op->id, device);
1505             op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);
1506 
1507             /* TODO check devices to verify watchdog will be in use */
1508         } else if(stonith_watchdog_timeout_ms > 0
1509                   && safe_str_eq(peer->host, op->target)
1510                   && safe_str_neq(op->action, "on")) {
1511             crm_notice("Waiting %ds for %s to self-fence (%s) for %s.%.8s (%p)",
1512                        stonith_watchdog_timeout_ms/1000, op->target,
1513                        op->action, op->client_name, op->id, device);
1514             op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);
1515 
1516         } else {
1517             op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op);
1518         }
1519 
1520 
1521         send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE);
1522         peer->tried = TRUE;
1523         free_xml(remote_op);
1524         return;
1525 
1526     } else if (op->phase == st_phase_on) {
1527         /* A remapped "on" cannot be executed, but the node was already
1528          * turned off successfully, so ignore the error and continue.
1529          */
1530         crm_warn("Ignoring %s 'on' failure (no capable peers) for %s after successful 'off'",
1531                  device, op->target);
1532         advance_op_topology(op, device, NULL, pcmk_ok);
1533         return;
1534 
1535     } else if (op->owner == FALSE) {
1536         crm_err("Fencing (%s) of %s for %s is not ours to control",
1537                 op->action, op->target, op->client_name);
1538 
1539     } else if (op->query_timer == 0) {
1540         /* We've exhausted all available peers */
1541         crm_info("No remaining peers capable of fencing (%s) %s for %s (%d)",
1542                  op->target, op->action, op->client_name, op->state);
1543         CRM_LOG_ASSERT(op->state < st_done);
1544         remote_op_timeout(op);
1545 
1546     } else if(op->replies >= op->replies_expected || op->replies >= fencing_active_peers()) {
1547         int rc = -EHOSTUNREACH;
1548 
1549         /* if the operation never left the query state,
1550          * but we have all the expected replies, then no devices
1551          * are available to execute the fencing operation. */
1552 
1553         if(stonith_watchdog_timeout_ms && (device == NULL || safe_str_eq(device, "watchdog"))) {
1554             crm_notice("Waiting %ds for %s to self-fence (%s) for %s.%.8s (%p)",
1555                      stonith_watchdog_timeout_ms/1000, op->target,
1556                      op->action, op->client_name, op->id, device);
1557 
1558             op->op_timer_one = g_timeout_add(stonith_watchdog_timeout_ms, remote_op_watchdog_done, op);
1559             return;
1560         }
1561 
1562         if (op->state == st_query) {
1563            crm_info("No peers (out of %d) have devices capable of fencing (%s) %s for %s (%d)",
1564                    op->replies, op->action, op->target, op->client_name,
1565                    op->state);
1566 
1567             rc = -ENODEV;
1568         } else {
1569            crm_info("No peers (out of %d) are capable of fencing (%s) %s for %s (%d)",
1570                    op->replies, op->action, op->target, op->client_name,
1571                    op->state);
1572         }
1573 
1574         op->state = st_failed;
1575         remote_op_done(op, NULL, rc, FALSE);
1576 
1577     } else if (device) {
1578         crm_info("Waiting for additional peers capable of fencing (%s) %s with %s for %s.%.8s",
1579                  op->action, op->target, device, op->client_name, op->id);
1580     } else {
1581         crm_info("Waiting for additional peers capable of fencing (%s) %s for %s%.8s",
1582                  op->action, op->target, op->client_name, op->id);
1583     }
1584 }
1585 
1586 /*!
1587  * \internal
1588  * \brief Comparison function for sorting query results
1589  *
1590  * \param[in] a  GList item to compare
1591  * \param[in] b  GList item to compare
1592  *
1593  * \return Per the glib documentation, "a negative integer if the first value
1594  *         comes before the second, 0 if they are equal, or a positive integer
1595  *         if the first value comes after the second."
1596  */
1597 static gint
1598 sort_peers(gconstpointer a, gconstpointer b)
     /*  */
1599 {
1600     const st_query_result_t *peer_a = a;
1601     const st_query_result_t *peer_b = b;
1602 
1603     return (peer_b->ndevices - peer_a->ndevices);
1604 }
1605 
1606 /*!
1607  * \internal
1608  * \brief Determine if all the devices in the topology are found or not
1609  */
1610 static gboolean
1611 all_topology_devices_found(remote_fencing_op_t * op)
     /*  */
1612 {
1613     GListPtr device = NULL;
1614     GListPtr iter = NULL;
1615     device_properties_t *match = NULL;
1616     stonith_topology_t *tp = NULL;
1617     gboolean skip_target = FALSE;
1618     int i;
1619 
1620     tp = find_topology_for_host(op->target);
1621     if (!tp) {
1622         return FALSE;
1623     }
1624     if (safe_str_eq(op->action, "off") || safe_str_eq(op->action, "reboot")) {
1625         /* Don't count the devices on the target node if we are killing
1626          * the target node. */
1627         skip_target = TRUE;
1628     }
1629 
1630     for (i = 0; i < ST_LEVEL_MAX; i++) {
1631         for (device = tp->levels[i]; device; device = device->next) {
1632             match = NULL;
1633             for (iter = op->query_results; iter && !match; iter = iter->next) {
1634                 st_query_result_t *peer = iter->data;
1635 
1636                 if (skip_target && safe_str_eq(peer->host, op->target)) {
1637                     continue;
1638                 }
1639                 match = find_peer_device(op, peer, device->data);
1640             }
1641             if (!match) {
1642                 return FALSE;
1643             }
1644         }
1645     }
1646 
1647     return TRUE;
1648 }
1649 
1650 /*!
1651  * \internal
1652  * \brief Parse action-specific device properties from XML
1653  *
1654  * \param[in]     msg     XML element containing the properties
1655  * \param[in]     peer    Name of peer that sent XML (for logs)
1656  * \param[in]     device  Device ID (for logs)
1657  * \param[in]     action  Action the properties relate to (for logs)
1658  * \param[in]     phase   Phase the properties relate to
1659  * \param[in,out] props   Device properties to update
1660  */
1661 static void
1662 parse_action_specific(xmlNode *xml, const char *peer, const char *device,
     /*  */
1663                       const char *action, remote_fencing_op_t *op,
1664                       enum st_remap_phase phase, device_properties_t *props)
1665 {
1666     props->custom_action_timeout[phase] = 0;
1667     crm_element_value_int(xml, F_STONITH_ACTION_TIMEOUT,
1668                           &props->custom_action_timeout[phase]);
1669     if (props->custom_action_timeout[phase]) {
1670         crm_trace("Peer %s with device %s returned %s action timeout %d",
1671                   peer, device, action, props->custom_action_timeout[phase]);
1672     }
1673 
1674     props->delay_max[phase] = 0;
1675     crm_element_value_int(xml, F_STONITH_DELAY_MAX, &props->delay_max[phase]);
1676     if (props->delay_max[phase]) {
1677         crm_trace("Peer %s with device %s returned maximum of random delay %d for %s",
1678                   peer, device, props->delay_max[phase], action);
1679     }
1680 
1681     props->delay_base[phase] = 0;
1682     crm_element_value_int(xml, F_STONITH_DELAY_BASE, &props->delay_base[phase]);
1683     if (props->delay_base[phase]) {
1684         crm_trace("Peer %s with device %s returned base delay %d for %s",
1685                   peer, device, props->delay_base[phase], action);
1686     }
1687 
1688     /* Handle devices with automatic unfencing */
1689     if (safe_str_eq(action, "on")) {
1690         int required = 0;
1691 
1692         crm_element_value_int(xml, F_STONITH_DEVICE_REQUIRED, &required);
1693         if (required) {
1694             crm_trace("Peer %s requires device %s to execute for action %s",
1695                       peer, device, action);
1696             add_required_device(op, device);
1697         }
1698     }
1699 
1700     /* If a reboot is remapped to off+on, it's possible that a node is allowed
1701      * to perform one action but not another.
1702      */
1703     if (crm_is_true(crm_element_value(xml, F_STONITH_ACTION_DISALLOWED))) {
1704         props->disallowed[phase] = TRUE;
1705         crm_trace("Peer %s is disallowed from executing %s for device %s",
1706                   peer, action, device);
1707     }
1708 }
1709 
1710 /*!
1711  * \internal
1712  * \brief Parse one device's properties from peer's XML query reply
1713  *
1714  * \param[in]     xml       XML node containing device properties
1715  * \param[in,out] op        Operation that query and reply relate to
1716  * \param[in,out] result    Peer's results
1717  * \param[in]     device    ID of device being parsed
1718  */
1719 static void
1720 add_device_properties(xmlNode *xml, remote_fencing_op_t *op,
     /*  */
1721                       st_query_result_t *result, const char *device)
1722 {
1723     xmlNode *child;
1724     int verified = 0;
1725     device_properties_t *props = calloc(1, sizeof(device_properties_t));
1726 
1727     /* Add a new entry to this result's devices list */
1728     CRM_ASSERT(props != NULL);
1729     g_hash_table_insert(result->devices, strdup(device), props);
1730 
1731     /* Peers with verified (monitored) access will be preferred */
1732     crm_element_value_int(xml, F_STONITH_DEVICE_VERIFIED, &verified);
1733     if (verified) {
1734         crm_trace("Peer %s has confirmed a verified device %s",
1735                   result->host, device);
1736         props->verified = TRUE;
1737     }
1738 
1739     /* Parse action-specific device properties */
1740     parse_action_specific(xml, result->host, device, op_requested_action(op),
1741                           op, st_phase_requested, props);
1742     for (child = __xml_first_child(xml); child != NULL; child = __xml_next(child)) {
1743         /* Replies for "reboot" operations will include the action-specific
1744          * values for "off" and "on" in child elements, just in case the reboot
1745          * winds up getting remapped.
1746          */
1747         if (safe_str_eq(ID(child), "off")) {
1748             parse_action_specific(child, result->host, device, "off",
1749                                   op, st_phase_off, props);
1750         } else if (safe_str_eq(ID(child), "on")) {
1751             parse_action_specific(child, result->host, device, "on",
1752                                   op, st_phase_on, props);
1753         }
1754     }
1755 }
1756 
1757 /*!
1758  * \internal
1759  * \brief Parse a peer's XML query reply and add it to operation's results
1760  *
1761  * \param[in,out] op        Operation that query and reply relate to
1762  * \param[in]     host      Name of peer that sent this reply
1763  * \param[in]     ndevices  Number of devices expected in reply
1764  * \param[in]     xml       XML node containing device list
1765  *
1766  * \return Newly allocated result structure with parsed reply
1767  */
1768 static st_query_result_t *
1769 add_result(remote_fencing_op_t *op, const char *host, int ndevices, xmlNode *xml)
     /*  */
1770 {
1771     st_query_result_t *result = calloc(1, sizeof(st_query_result_t));
1772     xmlNode *child;
1773 
1774     CRM_CHECK(result != NULL, return NULL);
1775     result->host = strdup(host);
1776     result->devices = crm_str_table_new();
1777 
1778     /* Each child element describes one capable device available to the peer */
1779     for (child = __xml_first_child(xml); child != NULL; child = __xml_next(child)) {
1780         const char *device = ID(child);
1781 
1782         if (device) {
1783             add_device_properties(child, op, result, device);
1784         }
1785     }
1786 
1787     result->ndevices = g_hash_table_size(result->devices);
1788     CRM_CHECK(ndevices == result->ndevices,
1789               crm_err("Query claimed to have %d devices but %d found",
1790                       ndevices, result->ndevices));
1791 
1792     op->query_results = g_list_insert_sorted(op->query_results, result, sort_peers);
1793     return result;
1794 }
1795 
1796 /*!
1797  * \internal
1798  * \brief Handle a peer's reply to our fencing query
1799  *
1800  * Parse a query result from XML and store it in the remote operation
1801  * table, and when enough replies have been received, issue a fencing request.
1802  *
1803  * \param[in] msg  XML reply received
1804  *
1805  * \return pcmk_ok on success, -errno on error
1806  *
1807  * \note See initiate_remote_stonith_op() for how the XML query was initially
1808  *       formed, and stonith_query() for how the peer formed its XML reply.
1809  */
1810 int
1811 process_remote_stonith_query(xmlNode * msg)
     /*  */
1812 {
1813     int ndevices = 0;
1814     gboolean host_is_target = FALSE;
1815     gboolean have_all_replies = FALSE;
1816     const char *id = NULL;
1817     const char *host = NULL;
1818     remote_fencing_op_t *op = NULL;
1819     st_query_result_t *result = NULL;
1820     uint32_t replies_expected;
1821     xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
1822 
1823     CRM_CHECK(dev != NULL, return -EPROTO);
1824 
1825     id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
1826     CRM_CHECK(id != NULL, return -EPROTO);
1827 
1828     dev = get_xpath_object("//@" F_STONITH_AVAILABLE_DEVICES, msg, LOG_ERR);
1829     CRM_CHECK(dev != NULL, return -EPROTO);
1830     crm_element_value_int(dev, F_STONITH_AVAILABLE_DEVICES, &ndevices);
1831 
1832     op = g_hash_table_lookup(remote_op_list, id);
1833     if (op == NULL) {
1834         crm_debug("Received query reply for unknown or expired operation %s",
1835                   id);
1836         return -EOPNOTSUPP;
1837     }
1838 
1839     replies_expected = QB_MIN(op->replies_expected, fencing_active_peers());
1840     if ((++op->replies >= replies_expected) && (op->state == st_query)) {
1841         have_all_replies = TRUE;
1842     }
1843     host = crm_element_value(msg, F_ORIG);
1844     host_is_target = safe_str_eq(host, op->target);
1845 
1846     crm_info("Query result %d of %d from %s for %s/%s (%d devices) %s",
1847              op->replies, replies_expected, host,
1848              op->target, op->action, ndevices, id);
1849     if (ndevices > 0) {
1850         result = add_result(op, host, ndevices, dev);
1851     }
1852 
1853     if (is_set(op->call_options, st_opt_topology)) {
1854         /* If we start the fencing before all the topology results are in,
1855          * it is possible fencing levels will be skipped because of the missing
1856          * query results. */
1857         if (op->state == st_query && all_topology_devices_found(op)) {
1858             /* All the query results are in for the topology, start the fencing ops. */
1859             crm_trace("All topology devices found");
1860             call_remote_stonith(op, result);
1861 
1862         } else if (have_all_replies) {
1863             crm_info("All topology query replies have arrived, continuing (%d expected/%d received) ",
1864                      replies_expected, op->replies);
1865             call_remote_stonith(op, NULL);
1866         }
1867 
1868     } else if (op->state == st_query) {
1869         int nverified = count_peer_devices(op, result, TRUE);
1870 
1871         /* We have a result for a non-topology fencing op that looks promising,
1872          * go ahead and start fencing before query timeout */
1873         if (result && (host_is_target == FALSE) && nverified) {
1874             /* we have a verified device living on a peer that is not the target */
1875             crm_trace("Found %d verified devices", nverified);
1876             call_remote_stonith(op, result);
1877 
1878         } else if (have_all_replies) {
1879             crm_info("All query replies have arrived, continuing (%d expected/%d received) ",
1880                      replies_expected, op->replies);
1881             call_remote_stonith(op, NULL);
1882 
1883         } else {
1884             crm_trace("Waiting for more peer results before launching fencing operation");
1885         }
1886 
1887     } else if (result && (op->state == st_done)) {
1888         crm_info("Discarding query result from %s (%d devices): Operation is in state %d",
1889                  result->host, result->ndevices, op->state);
1890     }
1891 
1892     return pcmk_ok;
1893 }
1894 
1895 /*!
1896  * \internal
1897  * \brief Handle a peer's reply to a fencing request
1898  *
1899  * Parse a fencing reply from XML, and either finalize the operation
1900  * or attempt another device as appropriate.
1901  *
1902  * \param[in] msg  XML reply received
1903  *
1904  * \return pcmk_ok on success, -errno on error
1905  */
1906 int
1907 process_remote_stonith_exec(xmlNode * msg)
     /*  */
1908 {
1909     int rc = 0;
1910     const char *id = NULL;
1911     const char *device = NULL;
1912     remote_fencing_op_t *op = NULL;
1913     xmlNode *dev = get_xpath_object("//@" F_STONITH_REMOTE_OP_ID, msg, LOG_ERR);
1914 
1915     CRM_CHECK(dev != NULL, return -EPROTO);
1916 
1917     id = crm_element_value(dev, F_STONITH_REMOTE_OP_ID);
1918     CRM_CHECK(id != NULL, return -EPROTO);
1919 
1920     dev = get_xpath_object("//@" F_STONITH_RC, msg, LOG_ERR);
1921     CRM_CHECK(dev != NULL, return -EPROTO);
1922 
1923     crm_element_value_int(dev, F_STONITH_RC, &rc);
1924 
1925     device = crm_element_value(dev, F_STONITH_DEVICE);
1926 
1927     if (remote_op_list) {
1928         op = g_hash_table_lookup(remote_op_list, id);
1929     }
1930 
1931     if (op == NULL && rc == pcmk_ok) {
1932         /* Record successful fencing operations */
1933         const char *client_id = crm_element_value(dev, F_STONITH_CLIENTID);
1934 
1935         op = create_remote_stonith_op(client_id, dev, TRUE);
1936     }
1937 
1938     if (op == NULL) {
1939         /* Could be for an event that began before we started */
1940         /* TODO: Record the op for later querying */
1941         crm_info("Received peer result of unknown or expired operation %s", id);
1942         return -EOPNOTSUPP;
1943     }
1944 
1945     if (op->devices && device && safe_str_neq(op->devices->data, device)) {
1946         crm_err("Received outdated reply for device %s (instead of %s) to "
1947                 "fence (%s) %s. Operation already timed out at peer level.",
1948                 device, op->devices->data, op->action, op->target);
1949         return rc;
1950     }
1951 
1952     if (safe_str_eq(crm_element_value(msg, F_SUBTYPE), "broadcast")) {
1953         crm_debug("Marking call to %s for %s on behalf of %s@%s.%.8s: %s (%d)",
1954                   op->action, op->target, op->client_name, op->id, op->originator,
1955                   pcmk_strerror(rc), rc);
1956         if (rc == pcmk_ok) {
1957             op->state = st_done;
1958         } else {
1959             op->state = st_failed;
1960         }
1961         remote_op_done(op, msg, rc, FALSE);
1962         return pcmk_ok;
1963     } else if (safe_str_neq(op->originator, stonith_our_uname)) {
1964         /* If this isn't a remote level broadcast, and we are not the
1965          * originator of the operation, we should not be receiving this msg. */
1966         crm_err
1967             ("%s received non-broadcast fencing result for operation it does not own (device %s targeting %s)",
1968              stonith_our_uname, device, op->target);
1969         return rc;
1970     }
1971 
1972     if (is_set(op->call_options, st_opt_topology)) {
1973         const char *device = crm_element_value(msg, F_STONITH_DEVICE);
1974 
1975         crm_notice("Call to %s for '%s %s' on behalf of %s@%s: %s (%d)",
1976                    device, op->target, op->action, op->client_name, op->originator,
1977                    pcmk_strerror(rc), rc);
1978 
1979         /* We own the op, and it is complete. broadcast the result to all nodes
1980          * and notify our local clients. */
1981         if (op->state == st_done) {
1982             remote_op_done(op, msg, rc, FALSE);
1983             return rc;
1984         }
1985 
1986         if ((op->phase == 2) && (rc != pcmk_ok)) {
1987             /* A remapped "on" failed, but the node was already turned off
1988              * successfully, so ignore the error and continue.
1989              */
1990             crm_warn("Ignoring %s 'on' failure (exit code %d) for %s after successful 'off'",
1991                      device, rc, op->target);
1992             rc = pcmk_ok;
1993         }
1994 
1995         if (rc == pcmk_ok) {
1996             /* An operation completed successfully. Try another device if
1997              * necessary, otherwise mark the operation as done. */
1998             advance_op_topology(op, device, msg, rc);
1999             return rc;
2000         } else {
2001             /* This device failed, time to try another topology level. If no other
2002              * levels are available, mark this operation as failed and report results. */
2003             if (stonith_topology_next(op) != pcmk_ok) {
2004                 op->state = st_failed;
2005                 remote_op_done(op, msg, rc, FALSE);
2006                 return rc;
2007             }
2008         }
2009     } else if (rc == pcmk_ok && op->devices == NULL) {
2010         crm_trace("All done for %s", op->target);
2011 
2012         op->state = st_done;
2013         remote_op_done(op, msg, rc, FALSE);
2014         return rc;
2015     } else if (rc == -ETIME && op->devices == NULL) {
2016         /* If the operation timed out don't bother retrying other peers. */
2017         op->state = st_failed;
2018         remote_op_done(op, msg, rc, FALSE);
2019         return rc;
2020     } else {
2021         /* fall-through and attempt other fencing action using another peer */
2022     }
2023 
2024     /* Retry on failure */
2025     crm_trace("Next for %s on behalf of %s@%s (rc was %d)", op->target, op->originator,
2026               op->client_name, rc);
2027     call_remote_stonith(op, NULL);
2028     return rc;
2029 }
2030 
2031 int
2032 stonith_fence_history(xmlNode * msg, xmlNode ** output)
     /*  */
2033 {
2034     int rc = 0;
2035     const char *target = NULL;
2036     xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, msg, LOG_TRACE);
2037     char *nodename = NULL;
2038 
2039     if (dev) {
2040         int options = 0;
2041 
2042         target = crm_element_value(dev, F_STONITH_TARGET);
2043         crm_element_value_int(msg, F_STONITH_CALLOPTS, &options);
2044         if (target && (options & st_opt_cs_nodeid)) {
2045             int nodeid = crm_atoi(target, NULL);
2046 
2047             nodename = stonith_get_peer_name(nodeid);
2048             if (nodename) {
2049                 target = nodename;
2050             }
2051         }
2052     }
2053 
2054     crm_trace("Looking for operations on %s in %p", target, remote_op_list);
2055 
2056     *output = create_xml_node(NULL, F_STONITH_HISTORY_LIST);
2057     if (remote_op_list) {
2058         GHashTableIter iter;
2059         remote_fencing_op_t *op = NULL;
2060 
2061         g_hash_table_iter_init(&iter, remote_op_list);
2062         while (g_hash_table_iter_next(&iter, NULL, (void **)&op)) {
2063             xmlNode *entry = NULL;
2064 
2065             if (target && strcmp(op->target, target) != 0) {
2066                 continue;
2067             }
2068 
2069             rc = 0;
2070             crm_trace("Attaching op %s", op->id);
2071             entry = create_xml_node(*output, STONITH_OP_EXEC);
2072             crm_xml_add(entry, F_STONITH_TARGET, op->target);
2073             crm_xml_add(entry, F_STONITH_ACTION, op->action);
2074             crm_xml_add(entry, F_STONITH_ORIGIN, op->originator);
2075             crm_xml_add(entry, F_STONITH_DELEGATE, op->delegate);
2076             crm_xml_add(entry, F_STONITH_CLIENTNAME, op->client_name);
2077             crm_xml_add_int(entry, F_STONITH_DATE, op->completed);
2078             crm_xml_add_int(entry, F_STONITH_STATE, op->state);
2079         }
2080     }
2081 
2082     free(nodename);
2083     return rc;
2084 }
2085 
2086 gboolean
2087 stonith_check_fence_tolerance(int tolerance, const char *target, const char *action)
     /*  */
2088 {
2089     GHashTableIter iter;
2090     time_t now = time(NULL);
2091     remote_fencing_op_t *rop = NULL;
2092 
2093     crm_trace("tolerance=%d, remote_op_list=%p", tolerance, remote_op_list);
2094 
2095     if (tolerance <= 0 || !remote_op_list || target == NULL || action == NULL) {
2096         return FALSE;
2097     }
2098 
2099     g_hash_table_iter_init(&iter, remote_op_list);
2100     while (g_hash_table_iter_next(&iter, NULL, (void **)&rop)) {
2101         if (strcmp(rop->target, target) != 0) {
2102             continue;
2103         } else if (rop->state != st_done) {
2104             continue;
2105         /* We don't have to worry about remapped reboots here
2106          * because if state is done, any remapping has been undone
2107          */
2108         } else if (strcmp(rop->action, action) != 0) {
2109             continue;
2110         } else if ((rop->completed + tolerance) < now) {
2111             continue;
2112         }
2113 
2114         crm_notice("Target %s was fenced (%s) less than %ds ago by %s on behalf of %s",
2115                    target, action, tolerance, rop->delegate, rop->originator);
2116         return TRUE;
2117     }
2118     return FALSE;
2119 }