root/tools/crm_mon.c

/* [previous][next][first][last][top][bottom][index][help] */

DEFINITIONS

This source file includes following definitions.
  1. all_includes
  2. default_includes
  3. find_section_bit
  4. apply_exclude
  5. apply_include
  6. apply_include_exclude
  7. user_include_exclude_cb
  8. include_exclude_cb
  9. as_cgi_cb
  10. as_html_cb
  11. as_simple_cb
  12. as_xml_cb
  13. fence_history_cb
  14. group_by_node_cb
  15. hide_headers_cb
  16. inactive_resources_cb
  17. no_curses_cb
  18. print_brief_cb
  19. print_detail_cb
  20. print_timing_cb
  21. reconnect_cb
  22. show_attributes_cb
  23. show_bans_cb
  24. show_failcounts_cb
  25. show_operations_cb
  26. show_tickets_cb
  27. use_cib_file_cb
  28. reconnect_after_timeout
  29. mon_cib_connection_destroy
  30. mon_shutdown
  31. mon_winresize
  32. fencing_connect
  33. cib_connect
  34. set_fencing_options
  35. pacemakerd_event_cb
  36. pacemakerd_status
  37. get_option_desc
  38. detect_user_input
  39. avoid_zombies
  40. build_arg_context
  41. add_output_args
  42. reconcile_output_format
  43. clean_up_on_connection_failure
  44. one_shot
  45. exit_on_invalid_cib
  46. main
  47. send_custom_trap
  48. handle_rsc_op
  49. mon_trigger_refresh
  50. crm_diff_update_v2
  51. crm_diff_update_v1
  52. crm_diff_update
  53. mon_refresh_display
  54. mon_st_callback_event
  55. refresh_after_event
  56. mon_st_callback_display
  57. clean_up_fencing_connection
  58. clean_up

   1 /*
   2  * Copyright 2004-2022 the Pacemaker project contributors
   3  *
   4  * The version control history for this file may have further details.
   5  *
   6  * This source code is licensed under the GNU General Public License version 2
   7  * or later (GPLv2+) WITHOUT ANY WARRANTY.
   8  */
   9 
  10 #include <crm_internal.h>
  11 
  12 #include <sys/param.h>
  13 
  14 #include <crm/crm.h>
  15 
  16 #include <stdint.h>
  17 #include <stdio.h>
  18 #include <sys/types.h>
  19 #include <sys/stat.h>
  20 #include <unistd.h>
  21 
  22 #include <stdlib.h>
  23 #include <errno.h>
  24 #include <fcntl.h>
  25 #include <libgen.h>
  26 #include <signal.h>
  27 #include <sys/utsname.h>
  28 
  29 #include <crm/msg_xml.h>
  30 #include <crm/services.h>
  31 #include <crm/lrmd.h>
  32 #include <crm/common/cmdline_internal.h>
  33 #include <crm/common/internal.h>  // pcmk__ends_with_ext()
  34 #include <crm/common/ipc.h>
  35 #include <crm/common/mainloop.h>
  36 #include <crm/common/output.h>
  37 #include <crm/common/output_internal.h>
  38 #include <crm/common/results.h>
  39 #include <crm/common/util.h>
  40 #include <crm/common/xml.h>
  41 #include <crm/common/xml_internal.h>
  42 
  43 #include <crm/cib/internal.h>
  44 #include <crm/pengine/status.h>
  45 #include <crm/pengine/internal.h>
  46 #include <pacemaker-internal.h>
  47 #include <crm/stonith-ng.h>
  48 #include <crm/fencing/internal.h>
  49 
  50 #include "crm_mon.h"
  51 
  52 #define SUMMARY "Provides a summary of cluster's current state.\n\n" \
  53                 "Outputs varying levels of detail in a number of different formats."
  54 
  55 /*
  56  * Definitions indicating which items to print
  57  */
  58 
  59 static uint32_t show;
  60 static uint32_t show_opts = pcmk_show_pending;
  61 
  62 /*
  63  * Definitions indicating how to output
  64  */
  65 
  66 static mon_output_format_t output_format = mon_output_unset;
  67 
  68 /* other globals */
  69 static GIOChannel *io_channel = NULL;
  70 static GMainLoop *mainloop = NULL;
  71 static guint reconnect_timer = 0;
  72 static mainloop_timer_t *refresh_timer = NULL;
  73 
  74 static cib_t *cib = NULL;
  75 static stonith_t *st = NULL;
  76 static xmlNode *current_cib = NULL;
  77 
  78 static GError *error = NULL;
  79 static pcmk__common_args_t *args = NULL;
  80 static pcmk__output_t *out = NULL;
  81 static GOptionContext *context = NULL;
  82 static gchar **processed_args = NULL;
  83 
  84 static time_t last_refresh = 0;
  85 volatile crm_trigger_t *refresh_trigger = NULL;
  86 
  87 static enum pcmk__fence_history fence_history = pcmk__fence_history_none;
  88 static gboolean on_remote_node = FALSE;
  89 static gboolean use_cib_native = FALSE;
  90 
  91 int interactive_fence_level = 0;
  92 
  93 static pcmk__supported_format_t formats[] = {
  94 #if CURSES_ENABLED
  95     CRM_MON_SUPPORTED_FORMAT_CURSES,
  96 #endif
  97     PCMK__SUPPORTED_FORMAT_HTML,
  98     PCMK__SUPPORTED_FORMAT_NONE,
  99     PCMK__SUPPORTED_FORMAT_TEXT,
 100     PCMK__SUPPORTED_FORMAT_XML,
 101     { NULL, NULL, NULL }
 102 };
 103 
 104 /* Define exit codes for monitoring-compatible output
 105  * For nagios plugins, the possibilities are
 106  * OK=0, WARN=1, CRIT=2, and UNKNOWN=3
 107  */
 108 #define MON_STATUS_WARN    CRM_EX_ERROR
 109 #define MON_STATUS_CRIT    CRM_EX_INVALID_PARAM
 110 #define MON_STATUS_UNKNOWN CRM_EX_UNIMPLEMENT_FEATURE
 111 
 112 #define RECONNECT_MSECS 5000
 113 
 114 struct {
 115     guint reconnect_ms;
 116     gboolean daemonize;
 117     gboolean fence_connect;
 118     gboolean one_shot;
 119     gboolean print_pending;
 120     gboolean show_bans;
 121     gboolean watch_fencing;
 122     char *pid_file;
 123     char *external_agent;
 124     char *external_recipient;
 125     char *neg_location_prefix;
 126     char *only_node;
 127     char *only_rsc;
 128     GSList *user_includes_excludes;
 129     GSList *includes_excludes;
 130 } options = {
 131     .fence_connect = TRUE,
 132     .reconnect_ms = RECONNECT_MSECS
 133 };
 134 
 135 static void clean_up_fencing_connection(void);
 136 static crm_exit_t clean_up(crm_exit_t exit_code);
 137 static void crm_diff_update(const char *event, xmlNode * msg);
 138 static void clean_up_on_connection_failure(int rc);
 139 static int mon_refresh_display(gpointer user_data);
 140 static int cib_connect(void);
 141 static int fencing_connect(void);
 142 static int pacemakerd_status(void);
 143 static void mon_st_callback_event(stonith_t * st, stonith_event_t * e);
 144 static void mon_st_callback_display(stonith_t * st, stonith_event_t * e);
 145 static void refresh_after_event(gboolean data_updated, gboolean enforce);
 146 
 147 static uint32_t
 148 all_includes(mon_output_format_t fmt) {
     /* [previous][next][first][last][top][bottom][index][help] */
 149     if (fmt == mon_output_monitor || fmt == mon_output_plain || fmt == mon_output_console) {
 150         return ~pcmk_section_options;
 151     } else {
 152         return pcmk_section_all;
 153     }
 154 }
 155 
 156 static uint32_t
 157 default_includes(mon_output_format_t fmt) {
     /* [previous][next][first][last][top][bottom][index][help] */
 158     switch (fmt) {
 159         case mon_output_monitor:
 160         case mon_output_plain:
 161         case mon_output_console:
 162             return pcmk_section_summary | pcmk_section_nodes | pcmk_section_resources |
 163                    pcmk_section_failures;
 164 
 165         case mon_output_xml:
 166         case mon_output_legacy_xml:
 167             return all_includes(fmt);
 168 
 169         case mon_output_html:
 170         case mon_output_cgi:
 171             return pcmk_section_summary | pcmk_section_nodes | pcmk_section_resources |
 172                    pcmk_section_failures;
 173 
 174         default:
 175             return 0;
 176     }
 177 }
 178 
 179 struct {
 180     const char *name;
 181     uint32_t bit;
 182 } sections[] = {
 183     { "attributes", pcmk_section_attributes },
 184     { "bans", pcmk_section_bans },
 185     { "counts", pcmk_section_counts },
 186     { "dc", pcmk_section_dc },
 187     { "failcounts", pcmk_section_failcounts },
 188     { "failures", pcmk_section_failures },
 189     { "fencing", pcmk_section_fencing_all },
 190     { "fencing-failed", pcmk_section_fence_failed },
 191     { "fencing-pending", pcmk_section_fence_pending },
 192     { "fencing-succeeded", pcmk_section_fence_worked },
 193     { "maint-mode", pcmk_section_maint_mode },
 194     { "nodes", pcmk_section_nodes },
 195     { "operations", pcmk_section_operations },
 196     { "options", pcmk_section_options },
 197     { "resources", pcmk_section_resources },
 198     { "stack", pcmk_section_stack },
 199     { "summary", pcmk_section_summary },
 200     { "tickets", pcmk_section_tickets },
 201     { "times", pcmk_section_times },
 202     { NULL }
 203 };
 204 
 205 static uint32_t
 206 find_section_bit(const char *name) {
     /* [previous][next][first][last][top][bottom][index][help] */
 207     for (int i = 0; sections[i].name != NULL; i++) {
 208         if (pcmk__str_eq(sections[i].name, name, pcmk__str_casei)) {
 209             return sections[i].bit;
 210         }
 211     }
 212 
 213     return 0;
 214 }
 215 
 216 static gboolean
 217 apply_exclude(const gchar *excludes, GError **error) {
     /* [previous][next][first][last][top][bottom][index][help] */
 218     char **parts = NULL;
 219     gboolean result = TRUE;
 220 
 221     parts = g_strsplit(excludes, ",", 0);
 222     for (char **s = parts; *s != NULL; s++) {
 223         uint32_t bit = find_section_bit(*s);
 224 
 225         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 226             show = 0;
 227         } else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
 228             show = all_includes(output_format);
 229         } else if (bit != 0) {
 230             show &= ~bit;
 231         } else {
 232             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 233                         "--exclude options: all, attributes, bans, counts, dc, "
 234                         "failcounts, failures, fencing, fencing-failed, "
 235                         "fencing-pending, fencing-succeeded, maint-mode, nodes, "
 236                         PCMK__VALUE_NONE ", operations, options, resources, "
 237                         "stack, summary, tickets, times");
 238             result = FALSE;
 239             break;
 240         }
 241     }
 242     g_strfreev(parts);
 243     return result;
 244 }
 245 
 246 static gboolean
 247 apply_include(const gchar *includes, GError **error) {
     /* [previous][next][first][last][top][bottom][index][help] */
 248     char **parts = NULL;
 249     gboolean result = TRUE;
 250 
 251     parts = g_strsplit(includes, ",", 0);
 252     for (char **s = parts; *s != NULL; s++) {
 253         uint32_t bit = find_section_bit(*s);
 254 
 255         if (pcmk__str_eq(*s, "all", pcmk__str_none)) {
 256             show = all_includes(output_format);
 257         } else if (pcmk__starts_with(*s, "bans")) {
 258             show |= pcmk_section_bans;
 259             if (options.neg_location_prefix != NULL) {
 260                 free(options.neg_location_prefix);
 261                 options.neg_location_prefix = NULL;
 262             }
 263 
 264             if (strlen(*s) > 4 && (*s)[4] == ':') {
 265                 options.neg_location_prefix = strdup(*s+5);
 266             }
 267         } else if (pcmk__str_any_of(*s, "default", "defaults", NULL)) {
 268             show |= default_includes(output_format);
 269         } else if (pcmk__str_eq(*s, PCMK__VALUE_NONE, pcmk__str_none)) {
 270             show = 0;
 271         } else if (bit != 0) {
 272             show |= bit;
 273         } else {
 274             g_set_error(error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
 275                         "--include options: all, attributes, bans[:PREFIX], counts, dc, "
 276                         "default, failcounts, failures, fencing, fencing-failed, "
 277                         "fencing-pending, fencing-succeeded, maint-mode, nodes, "
 278                         PCMK__VALUE_NONE ", operations, options, resources, "
 279                         "stack, summary, tickets, times");
 280             result = FALSE;
 281             break;
 282         }
 283     }
 284     g_strfreev(parts);
 285     return result;
 286 }
 287 
 288 static gboolean
 289 apply_include_exclude(GSList *lst, GError **error) {
     /* [previous][next][first][last][top][bottom][index][help] */
 290     gboolean rc = TRUE;
 291     GSList *node = lst;
 292 
 293     while (node != NULL) {
 294         char *s = node->data;
 295 
 296         if (pcmk__starts_with(s, "--include=")) {
 297             rc = apply_include(s+10, error);
 298         } else if (pcmk__starts_with(s, "-I=")) {
 299             rc = apply_include(s+3, error);
 300         } else if (pcmk__starts_with(s, "--exclude=")) {
 301             rc = apply_exclude(s+10, error);
 302         } else if (pcmk__starts_with(s, "-U=")) {
 303             rc = apply_exclude(s+3, error);
 304         }
 305 
 306         if (rc != TRUE) {
 307             break;
 308         }
 309 
 310         node = node->next;
 311     }
 312 
 313     return rc;
 314 }
 315 
 316 static gboolean
 317 user_include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 318     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 319 
 320     options.user_includes_excludes = g_slist_append(options.user_includes_excludes, s);
 321     return TRUE;
 322 }
 323 
 324 static gboolean
 325 include_exclude_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 326     char *s = crm_strdup_printf("%s=%s", option_name, optarg);
 327 
 328     options.includes_excludes = g_slist_append(options.includes_excludes, s);
 329     return TRUE;
 330 }
 331 
 332 static gboolean
 333 as_cgi_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 334     pcmk__str_update(&args->output_ty, "html");
 335     output_format = mon_output_cgi;
 336     options.one_shot = TRUE;
 337     return TRUE;
 338 }
 339 
 340 static gboolean
 341 as_html_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 342     pcmk__str_update(&args->output_dest, optarg);
 343     pcmk__str_update(&args->output_ty, "html");
 344     output_format = mon_output_html;
 345     umask(S_IWGRP | S_IWOTH);
 346     return TRUE;
 347 }
 348 
 349 static gboolean
 350 as_simple_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 351     pcmk__str_update(&args->output_ty, "text");
 352     output_format = mon_output_monitor;
 353     options.one_shot = TRUE;
 354     return TRUE;
 355 }
 356 
 357 static gboolean
 358 as_xml_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 359     pcmk__str_update(&args->output_ty, "xml");
 360     output_format = mon_output_legacy_xml;
 361     return TRUE;
 362 }
 363 
 364 static gboolean
 365 fence_history_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 366     if (optarg == NULL) {
 367         interactive_fence_level = 2;
 368     } else {
 369         pcmk__scan_min_int(optarg, &interactive_fence_level, 0);
 370     }
 371 
 372     switch (interactive_fence_level) {
 373         case 3:
 374             options.fence_connect = TRUE;
 375             fence_history = pcmk__fence_history_full;
 376             return include_exclude_cb("--include", "fencing", data, err);
 377 
 378         case 2:
 379             options.fence_connect = TRUE;
 380             fence_history = pcmk__fence_history_full;
 381             return include_exclude_cb("--include", "fencing", data, err);
 382 
 383         case 1:
 384             options.fence_connect = TRUE;
 385             fence_history = pcmk__fence_history_full;
 386             return include_exclude_cb("--include", "fencing-failed,fencing-pending", data, err);
 387 
 388         case 0:
 389             options.fence_connect = FALSE;
 390             fence_history = pcmk__fence_history_none;
 391             return include_exclude_cb("--exclude", "fencing", data, err);
 392 
 393         default:
 394             g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Fence history must be 0-3");
 395             return FALSE;
 396     }
 397 }
 398 
 399 static gboolean
 400 group_by_node_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 401     show_opts |= pcmk_show_rscs_by_node;
 402     return TRUE;
 403 }
 404 
 405 static gboolean
 406 hide_headers_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 407     return user_include_exclude_cb("--exclude", "summary", data, err);
 408 }
 409 
 410 static gboolean
 411 inactive_resources_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 412     show_opts |= pcmk_show_inactive_rscs;
 413     return TRUE;
 414 }
 415 
 416 static gboolean
 417 no_curses_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 418     output_format = mon_output_plain;
 419     return TRUE;
 420 }
 421 
 422 static gboolean
 423 print_brief_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 424     show_opts |= pcmk_show_brief;
 425     return TRUE;
 426 }
 427 
 428 static gboolean
 429 print_detail_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 430     show_opts |= pcmk_show_details;
 431     return TRUE;
 432 }
 433 
 434 static gboolean
 435 print_timing_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 436     show_opts |= pcmk_show_timing;
 437     return user_include_exclude_cb("--include", "operations", data, err);
 438 }
 439 
 440 static gboolean
 441 reconnect_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 442     int rc = crm_get_msec(optarg);
 443 
 444     if (rc == -1) {
 445         g_set_error(err, PCMK__EXITC_ERROR, CRM_EX_INVALID_PARAM, "Invalid value for -i: %s", optarg);
 446         return FALSE;
 447     } else {
 448         options.reconnect_ms = crm_parse_interval_spec(optarg);
 449     }
 450 
 451     return TRUE;
 452 }
 453 
 454 static gboolean
 455 show_attributes_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 456     return user_include_exclude_cb("--include", "attributes", data, err);
 457 }
 458 
 459 static gboolean
 460 show_bans_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 461     if (optarg != NULL) {
 462         char *s = crm_strdup_printf("bans:%s", optarg);
 463         gboolean rc = user_include_exclude_cb("--include", s, data, err);
 464         free(s);
 465         return rc;
 466     } else {
 467         return user_include_exclude_cb("--include", "bans", data, err);
 468     }
 469 }
 470 
 471 static gboolean
 472 show_failcounts_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 473     return user_include_exclude_cb("--include", "failcounts", data, err);
 474 }
 475 
 476 static gboolean
 477 show_operations_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 478     return user_include_exclude_cb("--include", "failcounts,operations", data, err);
 479 }
 480 
 481 static gboolean
 482 show_tickets_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 483     return user_include_exclude_cb("--include", "tickets", data, err);
 484 }
 485 
 486 static gboolean
 487 use_cib_file_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
     /* [previous][next][first][last][top][bottom][index][help] */
 488     setenv("CIB_file", optarg, 1);
 489     options.one_shot = TRUE;
 490     return TRUE;
 491 }
 492 
 493 #define INDENT "                                    "
 494 
 495 /* *INDENT-OFF* */
 496 static GOptionEntry addl_entries[] = {
 497     { "interval", 'i', 0, G_OPTION_ARG_CALLBACK, reconnect_cb,
 498       "Update frequency (default is 5 seconds)",
 499       "TIMESPEC" },
 500 
 501     { "one-shot", '1', 0, G_OPTION_ARG_NONE, &options.one_shot,
 502       "Display the cluster status once on the console and exit",
 503       NULL },
 504 
 505     { "daemonize", 'd', 0, G_OPTION_ARG_NONE, &options.daemonize,
 506       "Run in the background as a daemon.\n"
 507       INDENT "Requires at least one of --output-to and --external-agent.",
 508       NULL },
 509 
 510     { "pid-file", 'p', 0, G_OPTION_ARG_FILENAME, &options.pid_file,
 511       "(Advanced) Daemon pid file location",
 512       "FILE" },
 513 
 514     { "external-agent", 'E', 0, G_OPTION_ARG_FILENAME, &options.external_agent,
 515       "A program to run when resource operations take place",
 516       "FILE" },
 517 
 518     { "external-recipient", 'e', 0, G_OPTION_ARG_STRING, &options.external_recipient,
 519       "A recipient for your program (assuming you want the program to send something to someone).",
 520       "RCPT" },
 521 
 522     { "watch-fencing", 'W', 0, G_OPTION_ARG_NONE, &options.watch_fencing,
 523       "Listen for fencing events. For use with --external-agent.",
 524       NULL },
 525 
 526     { "xml-file", 'x', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_CALLBACK, use_cib_file_cb,
 527       NULL,
 528       NULL },
 529 
 530     { NULL }
 531 };
 532 
 533 static GOptionEntry display_entries[] = {
 534     { "include", 'I', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 535       "A list of sections to include in the output.\n"
 536       INDENT "See `Output Control` help for more information.",
 537       "SECTION(s)" },
 538 
 539     { "exclude", 'U', 0, G_OPTION_ARG_CALLBACK, user_include_exclude_cb,
 540       "A list of sections to exclude from the output.\n"
 541       INDENT "See `Output Control` help for more information.",
 542       "SECTION(s)" },
 543 
 544     { "node", 0, 0, G_OPTION_ARG_STRING, &options.only_node,
 545       "When displaying information about nodes, show only what's related to the given\n"
 546       INDENT "node, or to all nodes tagged with the given tag",
 547       "NODE" },
 548 
 549     { "resource", 0, 0, G_OPTION_ARG_STRING, &options.only_rsc,
 550       "When displaying information about resources, show only what's related to the given\n"
 551       INDENT "resource, or to all resources tagged with the given tag",
 552       "RSC" },
 553 
 554     { "group-by-node", 'n', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, group_by_node_cb,
 555       "Group resources by node",
 556       NULL },
 557 
 558     { "inactive", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, inactive_resources_cb,
 559       "Display inactive resources",
 560       NULL },
 561 
 562     { "failcounts", 'f', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_failcounts_cb,
 563       "Display resource fail counts",
 564       NULL },
 565 
 566     { "operations", 'o', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_operations_cb,
 567       "Display resource operation history",
 568       NULL },
 569 
 570     { "timing-details", 't', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_timing_cb,
 571       "Display resource operation history with timing details",
 572       NULL },
 573 
 574     { "tickets", 'c', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_tickets_cb,
 575       "Display cluster tickets",
 576       NULL },
 577 
 578     { "fence-history", 'm', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, fence_history_cb,
 579       "Show fence history:\n"
 580       INDENT "0=off, 1=failures and pending (default without option),\n"
 581       INDENT "2=add successes (default without value for option),\n"
 582       INDENT "3=show full history without reduction to most recent of each flavor",
 583       "LEVEL" },
 584 
 585     { "neg-locations", 'L', G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK, show_bans_cb,
 586       "Display negative location constraints [optionally filtered by id prefix]",
 587       NULL },
 588 
 589     { "show-node-attributes", 'A', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, show_attributes_cb,
 590       "Display node attributes",
 591       NULL },
 592 
 593     { "hide-headers", 'D', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, hide_headers_cb,
 594       "Hide all headers",
 595       NULL },
 596 
 597     { "show-detail", 'R', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_detail_cb,
 598       "Show more details (node IDs, individual clone instances)",
 599       NULL },
 600 
 601     { "brief", 'b', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, print_brief_cb,
 602       "Brief output",
 603       NULL },
 604 
 605     { "pending", 'j', G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &options.print_pending,
 606       "Display pending state if 'record-pending' is enabled",
 607       NULL },
 608 
 609     { "simple-status", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_simple_cb,
 610       "Display the cluster status once as a simple one line output (suitable for nagios)",
 611       NULL },
 612 
 613     { NULL }
 614 };
 615 
 616 static GOptionEntry deprecated_entries[] = {
 617     { "as-html", 'h', G_OPTION_FLAG_FILENAME, G_OPTION_ARG_CALLBACK, as_html_cb,
 618       "Write cluster status to the named HTML file.\n"
 619       INDENT "Use --output-as=html --output-to=FILE instead.",
 620       "FILE" },
 621 
 622     { "as-xml", 'X', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_xml_cb,
 623       "Write cluster status as XML to stdout. This will enable one-shot mode.\n"
 624       INDENT "Use --output-as=xml instead.",
 625       NULL },
 626 
 627     { "disable-ncurses", 'N', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, no_curses_cb,
 628       "Disable the use of ncurses.\n"
 629       INDENT "Use --output-as=text instead.",
 630       NULL },
 631 
 632     { "web-cgi", 'w', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, as_cgi_cb,
 633       "Web mode with output suitable for CGI (preselected when run as *.cgi).\n"
 634       INDENT "Use --output-as=html --html-cgi instead.",
 635       NULL },
 636 
 637     { NULL }
 638 };
 639 /* *INDENT-ON* */
 640 
 641 /* Reconnect to the CIB and fencing agent after reconnect_ms has passed.  This sounds
 642  * like it would be more broadly useful, but only ever happens after a disconnect via
 643  * mon_cib_connection_destroy.
 644  */
 645 static gboolean
 646 reconnect_after_timeout(gpointer data)
     /* [previous][next][first][last][top][bottom][index][help] */
 647 {
 648 #if CURSES_ENABLED
 649     if (output_format == mon_output_console) {
 650         clear();
 651         refresh();
 652     }
 653 #endif
 654 
 655     out->info(out, "Reconnecting...");
 656     if (pacemakerd_status() == pcmk_rc_ok) {
 657         fencing_connect();
 658         if (cib_connect() == pcmk_rc_ok) {
 659             /* trigger redrawing the screen (needs reconnect_timer == 0) */
 660             reconnect_timer = 0;
 661             refresh_after_event(FALSE, TRUE);
 662             return G_SOURCE_REMOVE;
 663         }
 664     }
 665 
 666     reconnect_timer = g_timeout_add(options.reconnect_ms,
 667                                     reconnect_after_timeout, NULL);
 668     return G_SOURCE_REMOVE;
 669 }
 670 
 671 /* Called from various places when we are disconnected from the CIB or from the
 672  * fencing agent.  If the CIB connection is still valid, this function will also
 673  * attempt to sign off and reconnect.
 674  */
 675 static void
 676 mon_cib_connection_destroy(gpointer user_data)
     /* [previous][next][first][last][top][bottom][index][help] */
 677 {
 678     out->info(out, "Connection to the cluster-daemons terminated");
 679 
 680     if (refresh_timer != NULL) {
 681         /* we'll trigger a refresh after reconnect */
 682         mainloop_timer_stop(refresh_timer);
 683     }
 684     if (reconnect_timer) {
 685         /* we'll trigger a new reconnect-timeout at the end */
 686         g_source_remove(reconnect_timer);
 687         reconnect_timer = 0;
 688     }
 689     if (st) {
 690         /* the client API won't properly reconnect notifications
 691          * if they are still in the table - so remove them
 692          */
 693         clean_up_fencing_connection();
 694     }
 695     if (cib) {
 696         cib->cmds->signoff(cib);
 697         reconnect_timer = g_timeout_add(options.reconnect_ms,
 698                                         reconnect_after_timeout, NULL);
 699     }
 700     return;
 701 }
 702 
 703 /* Signal handler installed into the mainloop for normal program shutdown */
 704 static void
 705 mon_shutdown(int nsig)
     /* [previous][next][first][last][top][bottom][index][help] */
 706 {
 707     clean_up(CRM_EX_OK);
 708 }
 709 
 710 #if CURSES_ENABLED
 711 static volatile sighandler_t ncurses_winch_handler;
 712 
 713 /* Signal handler installed the regular way (not into the main loop) for when
 714  * the screen is resized.  Commonly, this happens when running in an xterm and
 715  * the user changes its size.
 716  */
 717 static void
 718 mon_winresize(int nsig)
     /* [previous][next][first][last][top][bottom][index][help] */
 719 {
 720     static int not_done;
 721     int lines = 0, cols = 0;
 722 
 723     if (!not_done++) {
 724         if (ncurses_winch_handler)
 725             /* the original ncurses WINCH signal handler does the
 726              * magic of retrieving the new window size;
 727              * otherwise, we'd have to use ioctl or tgetent */
 728             (*ncurses_winch_handler) (SIGWINCH);
 729         getmaxyx(stdscr, lines, cols);
 730         resizeterm(lines, cols);
 731         /* Alert the mainloop code we'd like the refresh_trigger to run next
 732          * time the mainloop gets around to checking.
 733          */
 734         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
 735     }
 736     not_done--;
 737 }
 738 #endif
 739 
 740 static int
 741 fencing_connect(void)
     /* [previous][next][first][last][top][bottom][index][help] */
 742 {
 743     int rc = pcmk_ok;
 744 
 745     if (options.fence_connect && st == NULL) {
 746         st = stonith_api_new();
 747     }
 748 
 749     if (!options.fence_connect || st == NULL || st->state != stonith_disconnected) {
 750         return rc;
 751     }
 752 
 753     rc = st->cmds->connect(st, crm_system_name, NULL);
 754     if (rc == pcmk_ok) {
 755         crm_trace("Setting up stonith callbacks");
 756         if (options.watch_fencing) {
 757             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 758                                             mon_st_callback_event);
 759             st->cmds->register_notification(st, T_STONITH_NOTIFY_FENCE, mon_st_callback_event);
 760         } else {
 761             st->cmds->register_notification(st, T_STONITH_NOTIFY_DISCONNECT,
 762                                             mon_st_callback_display);
 763             st->cmds->register_notification(st, T_STONITH_NOTIFY_HISTORY, mon_st_callback_display);
 764         }
 765     } else {
 766         clean_up_fencing_connection();
 767     }
 768 
 769     return rc;
 770 }
 771 
 772 static int
 773 cib_connect(void)
     /* [previous][next][first][last][top][bottom][index][help] */
 774 {
 775     int rc = pcmk_rc_ok;
 776 
 777     CRM_CHECK(cib != NULL, return EINVAL);
 778 
 779     if (cib->state == cib_connected_query ||
 780         cib->state == cib_connected_command) {
 781         return rc;
 782     }
 783 
 784     crm_trace("Connecting to the CIB");
 785 
 786     rc = pcmk_legacy2rc(cib->cmds->signon(cib, crm_system_name, cib_query));
 787     if (rc != pcmk_rc_ok) {
 788         out->err(out, "Could not connect to the CIB: %s",
 789                  pcmk_rc_str(rc));
 790         return rc;
 791     }
 792 
 793     /* just show this if refresh is gonna remove all traces */
 794     if (output_format == mon_output_console) {
 795         out->info(out,"Waiting for CIB ...");
 796     }
 797 
 798     rc = pcmk_legacy2rc(cib->cmds->query(cib, NULL, &current_cib,
 799                                          cib_scope_local | cib_sync_call));
 800 
 801     if (rc == pcmk_rc_ok) {
 802         rc = pcmk_legacy2rc(cib->cmds->set_connection_dnotify(cib,
 803             mon_cib_connection_destroy));
 804         if (rc == EPROTONOSUPPORT) {
 805             out->err(out,
 806                      "Notification setup not supported, won't be "
 807                      "able to reconnect after failure");
 808             if (output_format == mon_output_console) {
 809                 sleep(2);
 810             }
 811             rc = pcmk_rc_ok;
 812         }
 813 
 814         if (rc == pcmk_rc_ok) {
 815             cib->cmds->del_notify_callback(cib, T_CIB_DIFF_NOTIFY,
 816                                            crm_diff_update);
 817             rc = pcmk_legacy2rc(cib->cmds->add_notify_callback(cib,
 818                                     T_CIB_DIFF_NOTIFY, crm_diff_update));
 819         }
 820 
 821         if (rc != pcmk_rc_ok) {
 822             out->err(out, "Notification setup failed, could not monitor CIB actions");
 823             cib__clean_up_connection(&cib);
 824             clean_up_fencing_connection();
 825         }
 826     }
 827     return rc;
 828 }
 829 
 830 /* This is used to set up the fencing options after the interactive UI has been stared.
 831  * fence_history_cb can't be used because it builds up a list of includes/excludes that
 832  * then have to be processed with apply_include_exclude and that could affect other
 833  * things.
 834  */
 835 static void
 836 set_fencing_options(int level)
     /* [previous][next][first][last][top][bottom][index][help] */
 837 {
 838     switch (level) {
 839         case 3:
 840             options.fence_connect = TRUE;
 841             fence_history = pcmk__fence_history_full;
 842             show |= pcmk_section_fencing_all;
 843             break;
 844 
 845         case 2:
 846             options.fence_connect = TRUE;
 847             fence_history = pcmk__fence_history_full;
 848             show |= pcmk_section_fencing_all;
 849             break;
 850 
 851         case 1:
 852             options.fence_connect = TRUE;
 853             fence_history = pcmk__fence_history_full;
 854             show |= pcmk_section_fence_failed | pcmk_section_fence_pending;
 855             break;
 856 
 857         default:
 858             interactive_fence_level = 0;
 859             options.fence_connect = FALSE;
 860             fence_history = pcmk__fence_history_none;
 861             show &= ~pcmk_section_fencing_all;
 862             break;
 863     }
 864 }
 865 
 866 /* Before trying to connect to fencer or cib check for state of
 867    pacemakerd - just no sense in trying till pacemakerd has
 868    taken care of starting all the sub-processes
 869 
 870    Only noteworthy thing to show here is when pacemakerd is
 871    waiting for startup-trigger from SBD.
 872  */
 873 static void
 874 pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
     /* [previous][next][first][last][top][bottom][index][help] */
 875                     enum pcmk_ipc_event event_type, crm_exit_t status,
 876                     void *event_data, void *user_data)
 877 {
 878     pcmk_pacemakerd_api_reply_t *reply = event_data;
 879     enum pcmk_pacemakerd_state *state =
 880         (enum pcmk_pacemakerd_state *) user_data;
 881 
 882     /* we are just interested in the latest reply */
 883     *state = pcmk_pacemakerd_state_invalid;
 884 
 885     switch (event_type) {
 886         case pcmk_ipc_event_reply:
 887             break;
 888 
 889         default:
 890             return;
 891     }
 892 
 893     if (status != CRM_EX_OK) {
 894         out->err(out, "Bad reply from pacemakerd: %s",
 895                  crm_exit_str(status));
 896         return;
 897     }
 898 
 899     if (reply->reply_type != pcmk_pacemakerd_reply_ping) {
 900         out->err(out, "Unknown reply type %d from pacemakerd",
 901                  reply->reply_type);
 902     } else {
 903         if ((reply->data.ping.last_good != (time_t) 0) &&
 904             (reply->data.ping.status == pcmk_rc_ok)) {
 905             *state = reply->data.ping.state;
 906         }
 907     }
 908 }
 909 
 910 static int
 911 pacemakerd_status(void)
     /* [previous][next][first][last][top][bottom][index][help] */
 912 {
 913     int rc = pcmk_rc_ok;
 914     pcmk_ipc_api_t *pacemakerd_api = NULL;
 915     enum pcmk_pacemakerd_state state = pcmk_pacemakerd_state_invalid;
 916 
 917     if (!use_cib_native) {
 918         /* we don't need fully functional pacemakerd otherwise */
 919         return rc;
 920     }
 921     if (cib != NULL &&
 922         (cib->state == cib_connected_query ||
 923          cib->state == cib_connected_command)) {
 924         /* As long as we have a cib-connection let's go with
 925          * that to fetch further cluster-status and avoid
 926          * unnecessary pings to pacemakerd.
 927          * If cluster is going down and fencer is down already
 928          * this will lead to a silently failing fencer reconnect.
 929          * On cluster startup we shouldn't see this situation
 930          * as first we do is wait for pacemakerd to report all
 931          * daemons running.
 932          */
 933         return rc;
 934     }
 935     rc = pcmk_new_ipc_api(&pacemakerd_api, pcmk_ipc_pacemakerd);
 936     if (pacemakerd_api == NULL) {
 937         out->err(out, "Could not connect to pacemakerd: %s",
 938                  pcmk_rc_str(rc));
 939         /* this is unrecoverable so return with rc we have */
 940         return rc;
 941     }
 942     pcmk_register_ipc_callback(pacemakerd_api, pacemakerd_event_cb, (void *) &state);
 943     rc = pcmk_connect_ipc(pacemakerd_api, pcmk_ipc_dispatch_poll);
 944     switch (rc) {
 945         case pcmk_rc_ok:
 946             rc = pcmk_pacemakerd_api_ping(pacemakerd_api, crm_system_name);
 947             if (rc == pcmk_rc_ok) {
 948                 rc = pcmk_poll_ipc(pacemakerd_api, options.reconnect_ms/2);
 949                 if (rc == pcmk_rc_ok) {
 950                     pcmk_dispatch_ipc(pacemakerd_api);
 951                     rc = ENOTCONN;
 952                     if ((output_format == mon_output_console) ||
 953                         (output_format == mon_output_plain)) {
 954                         switch (state) {
 955                             case pcmk_pacemakerd_state_running:
 956                                 rc = pcmk_rc_ok;
 957                                 break;
 958                             case pcmk_pacemakerd_state_starting_daemons:
 959                                 out->info(out,"Pacemaker daemons starting ...");
 960                                 break;
 961                             case pcmk_pacemakerd_state_wait_for_ping:
 962                                 out->info(out,"Waiting for startup-trigger from SBD ...");
 963                                 break;
 964                             case pcmk_pacemakerd_state_shutting_down:
 965                                 out->info(out,"Pacemaker daemons shutting down ...");
 966                                 /* try our luck maybe CIB is still accessible */
 967                                 rc = pcmk_rc_ok;
 968                                 break;
 969                             case pcmk_pacemakerd_state_shutdown_complete:
 970                                 /* assuming pacemakerd doesn't dispatch any pings after entering
 971                                 * that state unless it is waiting for SBD
 972                                 */
 973                                 out->info(out,"Pacemaker daemons shut down - reporting to SBD ...");
 974                                 break;
 975                             default:
 976                                 break;
 977                         }
 978                     } else {
 979                         switch (state) {
 980                             case pcmk_pacemakerd_state_running:
 981                                 rc = pcmk_rc_ok;
 982                                 break;
 983                             case pcmk_pacemakerd_state_shutting_down:
 984                                 /* try our luck maybe CIB is still accessible */
 985                                 rc = pcmk_rc_ok;
 986                                 break;
 987                             default:
 988                                 break;
 989                         }
 990                     }
 991                 }
 992             }
 993             break;
 994         case EREMOTEIO:
 995             rc = pcmk_rc_ok;
 996             on_remote_node = TRUE;
 997             /* just show this if refresh is gonna remove all traces */
 998             if (output_format == mon_output_console) {
 999                 out->info(out,
1000                     "Running on remote-node waiting to be connected by cluster ...");
1001             }
1002             break;
1003         default:
1004             break;
1005     }
1006     pcmk_free_ipc_api(pacemakerd_api);
1007     /* returning with ENOTCONN triggers a retry */
1008     return (rc == pcmk_rc_ok)?rc:ENOTCONN;
1009 }
1010 
1011 #if CURSES_ENABLED
1012 static const char *
1013 get_option_desc(char c)
     /* [previous][next][first][last][top][bottom][index][help] */
1014 {
1015     const char *desc = "No help available";
1016 
1017     for (GOptionEntry *entry = display_entries; entry != NULL; entry++) {
1018         if (entry->short_name == c) {
1019             desc = entry->description;
1020             break;
1021         }
1022     }
1023     return desc;
1024 }
1025 
1026 #define print_option_help(out, option, condition) \
1027     curses_formatted_printf(out, "%c %c: \t%s\n", ((condition)? '*': ' '), option, get_option_desc(option));
1028 
1029 /* This function is called from the main loop when there is something to be read
1030  * on stdin, like an interactive user's keystroke.  All it does is read the keystroke,
1031  * set flags (or show the page showing which keystrokes are valid), and redraw the
1032  * screen.  It does not do anything with connections to the CIB or fencing agent
1033  * agent what would happen in mon_refresh_display.
1034  */
1035 static gboolean
1036 detect_user_input(GIOChannel *channel, GIOCondition condition, gpointer user_data)
     /* [previous][next][first][last][top][bottom][index][help] */
1037 {
1038     int c;
1039     gboolean config_mode = FALSE;
1040 
1041     while (1) {
1042 
1043         /* Get user input */
1044         c = getchar();
1045 
1046         switch (c) {
1047             case 'm':
1048                 interactive_fence_level++;
1049                 if (interactive_fence_level > 3) {
1050                     interactive_fence_level = 0;
1051                 }
1052 
1053                 set_fencing_options(interactive_fence_level);
1054                 break;
1055             case 'c':
1056                 show ^= pcmk_section_tickets;
1057                 break;
1058             case 'f':
1059                 show ^= pcmk_section_failcounts;
1060                 break;
1061             case 'n':
1062                 show_opts ^= pcmk_show_rscs_by_node;
1063                 break;
1064             case 'o':
1065                 show ^= pcmk_section_operations;
1066                 if (!pcmk_is_set(show, pcmk_section_operations)) {
1067                     show_opts &= ~pcmk_show_timing;
1068                 }
1069                 break;
1070             case 'r':
1071                 show_opts ^= pcmk_show_inactive_rscs;
1072                 break;
1073             case 'R':
1074                 show_opts ^= pcmk_show_details;
1075 #ifdef PCMK__COMPAT_2_0
1076                 // Keep failed action output the same as 2.0.x
1077                 show_opts |= pcmk_show_failed_detail;
1078 #endif
1079                 break;
1080             case 't':
1081                 show_opts ^= pcmk_show_timing;
1082                 if (pcmk_is_set(show_opts, pcmk_show_timing)) {
1083                     show |= pcmk_section_operations;
1084                 }
1085                 break;
1086             case 'A':
1087                 show ^= pcmk_section_attributes;
1088                 break;
1089             case 'L':
1090                 show ^= pcmk_section_bans;
1091                 break;
1092             case 'D':
1093                 /* If any header is shown, clear them all, otherwise set them all */
1094                 if (pcmk_any_flags_set(show, pcmk_section_summary)) {
1095                     show &= ~pcmk_section_summary;
1096                 } else {
1097                     show |= pcmk_section_summary;
1098                 }
1099                 /* Regardless, we don't show options in console mode. */
1100                 show &= ~pcmk_section_options;
1101                 break;
1102             case 'b':
1103                 show_opts ^= pcmk_show_brief;
1104                 break;
1105             case 'j':
1106                 show_opts ^= pcmk_show_pending;
1107                 break;
1108             case '?':
1109                 config_mode = TRUE;
1110                 break;
1111             default:
1112                 /* All other keys just redraw the screen. */
1113                 goto refresh;
1114         }
1115 
1116         if (!config_mode)
1117             goto refresh;
1118 
1119         blank_screen();
1120 
1121         curses_formatted_printf(out, "%s", "Display option change mode\n");
1122         print_option_help(out, 'c', pcmk_is_set(show, pcmk_section_tickets));
1123         print_option_help(out, 'f', pcmk_is_set(show, pcmk_section_failcounts));
1124         print_option_help(out, 'n', pcmk_is_set(show_opts, pcmk_show_rscs_by_node));
1125         print_option_help(out, 'o', pcmk_is_set(show, pcmk_section_operations));
1126         print_option_help(out, 'r', pcmk_is_set(show_opts, pcmk_show_inactive_rscs));
1127         print_option_help(out, 't', pcmk_is_set(show_opts, pcmk_show_timing));
1128         print_option_help(out, 'A', pcmk_is_set(show, pcmk_section_attributes));
1129         print_option_help(out, 'L', pcmk_is_set(show, pcmk_section_bans));
1130         print_option_help(out, 'D', !pcmk_is_set(show, pcmk_section_summary));
1131 #ifdef PCMK__COMPAT_2_0
1132         print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details & ~pcmk_show_failed_detail));
1133 #else
1134         print_option_help(out, 'R', pcmk_any_flags_set(show_opts, pcmk_show_details));
1135 #endif
1136         print_option_help(out, 'b', pcmk_is_set(show_opts, pcmk_show_brief));
1137         print_option_help(out, 'j', pcmk_is_set(show_opts, pcmk_show_pending));
1138         curses_formatted_printf(out, "%d m: \t%s\n", interactive_fence_level, get_option_desc('m'));
1139         curses_formatted_printf(out, "%s", "\nToggle fields via field letter, type any other key to return\n");
1140     }
1141 
1142 refresh:
1143     refresh_after_event(FALSE, TRUE);
1144 
1145     return TRUE;
1146 }
1147 #endif
1148 
1149 // Basically crm_signal_handler(SIGCHLD, SIG_IGN) plus the SA_NOCLDWAIT flag
1150 static void
1151 avoid_zombies(void)
     /* [previous][next][first][last][top][bottom][index][help] */
1152 {
1153     struct sigaction sa;
1154 
1155     memset(&sa, 0, sizeof(struct sigaction));
1156     if (sigemptyset(&sa.sa_mask) < 0) {
1157         crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
1158         return;
1159     }
1160     sa.sa_handler = SIG_IGN;
1161     sa.sa_flags = SA_RESTART|SA_NOCLDWAIT;
1162     if (sigaction(SIGCHLD, &sa, NULL) < 0) {
1163         crm_warn("Cannot avoid zombies: %s", pcmk_rc_str(errno));
1164     }
1165 }
1166 
1167 static GOptionContext *
1168 build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
     /* [previous][next][first][last][top][bottom][index][help] */
1169     GOptionContext *context = NULL;
1170 
1171     GOptionEntry extra_prog_entries[] = {
1172         { "quiet", 'Q', 0, G_OPTION_ARG_NONE, &(args->quiet),
1173           "Be less descriptive in output.",
1174           NULL },
1175 
1176         { NULL }
1177     };
1178 
1179     const char *description = "Notes:\n\n"
1180                               "If this program is called as crm_mon.cgi, --output-as=html --html-cgi will\n"
1181                               "automatically be added to the command line arguments.\n\n"
1182                               "Time Specification:\n\n"
1183                               "The TIMESPEC in any command line option can be specified in many different\n"
1184                               "formats.  It can be just an integer number of seconds, a number plus units\n"
1185                               "(ms/msec/us/usec/s/sec/m/min/h/hr), or an ISO 8601 period specification.\n\n"
1186                               "Output Control:\n\n"
1187                               "By default, a certain list of sections are written to the output destination.\n"
1188                               "The default varies based on the output format - XML includes everything, while\n"
1189                               "other output formats will display less.  This list can be modified with the\n"
1190                               "--include and --exclude command line options.  Each option may be given multiple\n"
1191                               "times on the command line, and each can give a comma-separated list of sections.\n"
1192                               "The options are applied to the default set, from left to right as seen on the\n"
1193                               "command line.  For a list of valid sections, pass --include=list or --exclude=list.\n\n"
1194                               "Interactive Use:\n\n"
1195                               "When run interactively, crm_mon can be told to hide and display various sections\n"
1196                               "of output.  To see a help screen explaining the options, hit '?'.  Any key stroke\n"
1197                               "aside from those listed will cause the screen to refresh.\n\n"
1198                               "Examples:\n\n"
1199                               "Display the cluster status on the console with updates as they occur:\n\n"
1200                               "\tcrm_mon\n\n"
1201                               "Display the cluster status on the console just once then exit:\n\n"
1202                               "\tcrm_mon -1\n\n"
1203                               "Display your cluster status, group resources by node, and include inactive resources in the list:\n\n"
1204                               "\tcrm_mon --group-by-node --inactive\n\n"
1205                               "Start crm_mon as a background daemon and have it write the cluster status to an HTML file:\n\n"
1206                               "\tcrm_mon --daemonize --output-as html --output-to /path/to/docroot/filename.html\n\n"
1207                               "Start crm_mon and export the current cluster status as XML to stdout, then exit:\n\n"
1208                               "\tcrm_mon --output-as xml\n\n";
1209 
1210 #if CURSES_ENABLED
1211     context = pcmk__build_arg_context(args, "console (default), html, text, xml", group, NULL);
1212 #else
1213     context = pcmk__build_arg_context(args, "text (default), html, xml", group, NULL);
1214 #endif
1215     pcmk__add_main_args(context, extra_prog_entries);
1216     g_option_context_set_description(context, description);
1217 
1218     pcmk__add_arg_group(context, "display", "Display Options:",
1219                         "Show display options", display_entries);
1220     pcmk__add_arg_group(context, "additional", "Additional Options:",
1221                         "Show additional options", addl_entries);
1222     pcmk__add_arg_group(context, "deprecated", "Deprecated Options:",
1223                         "Show deprecated options", deprecated_entries);
1224 
1225     return context;
1226 }
1227 
1228 /* If certain format options were specified, we want to set some extra
1229  * options.  We can just process these like they were given on the
1230  * command line.
1231  */
1232 static void
1233 add_output_args(void) {
     /* [previous][next][first][last][top][bottom][index][help] */
1234     GError *err = NULL;
1235 
1236     if (output_format == mon_output_plain) {
1237         if (!pcmk__force_args(context, &err, "%s --text-fancy", g_get_prgname())) {
1238             g_propagate_error(&error, err);
1239             clean_up(CRM_EX_USAGE);
1240         }
1241     } else if (output_format == mon_output_cgi) {
1242         if (!pcmk__force_args(context, &err, "%s --html-cgi", g_get_prgname())) {
1243             g_propagate_error(&error, err);
1244             clean_up(CRM_EX_USAGE);
1245         }
1246     } else if (output_format == mon_output_xml) {
1247         if (!pcmk__force_args(context, &err, "%s --xml-simple-list --xml-substitute", g_get_prgname())) {
1248             g_propagate_error(&error, err);
1249             clean_up(CRM_EX_USAGE);
1250         }
1251     } else if (output_format == mon_output_legacy_xml) {
1252         output_format = mon_output_xml;
1253         if (!pcmk__force_args(context, &err, "%s --xml-legacy --xml-substitute", g_get_prgname())) {
1254             g_propagate_error(&error, err);
1255             clean_up(CRM_EX_USAGE);
1256         }
1257     }
1258 }
1259 
1260 /* Which output format to use could come from two places:  The --as-xml
1261  * style arguments we gave in deprecated_entries above, or the formatted output
1262  * arguments added by pcmk__register_formats.  If the latter were used,
1263  * output_format will be mon_output_unset.
1264  *
1265  * Call the callbacks as if those older style arguments were provided so
1266  * the various things they do get done.
1267  */
1268 static void
1269 reconcile_output_format(pcmk__common_args_t *args) {
     /* [previous][next][first][last][top][bottom][index][help] */
1270     gboolean retval = TRUE;
1271     GError *err = NULL;
1272 
1273     if (output_format != mon_output_unset) {
1274         return;
1275     }
1276 
1277     if (pcmk__str_eq(args->output_ty, "html", pcmk__str_casei)) {
1278         char *dest = NULL;
1279 
1280         pcmk__str_update(&dest, args->output_dest);
1281         retval = as_html_cb("h", dest, NULL, &err);
1282         free(dest);
1283     } else if (pcmk__str_eq(args->output_ty, "text", pcmk__str_casei)) {
1284         retval = no_curses_cb("N", NULL, NULL, &err);
1285     } else if (pcmk__str_eq(args->output_ty, "xml", pcmk__str_casei)) {
1286         pcmk__str_update(&args->output_ty, "xml");
1287         output_format = mon_output_xml;
1288     } else if (options.one_shot) {
1289         pcmk__str_update(&args->output_ty, "text");
1290         output_format = mon_output_plain;
1291     } else if (!options.daemonize && args->output_dest != NULL) {
1292         options.one_shot = TRUE;
1293         pcmk__str_update(&args->output_ty, "text");
1294         output_format = mon_output_plain;
1295     } else {
1296         /* Neither old nor new arguments were given, so set the default. */
1297         pcmk__str_update(&args->output_ty, "console");
1298         output_format = mon_output_console;
1299     }
1300 
1301     if (!retval) {
1302         g_propagate_error(&error, err);
1303         clean_up(CRM_EX_USAGE);
1304     }
1305 }
1306 
1307 static void
1308 clean_up_on_connection_failure(int rc)
     /* [previous][next][first][last][top][bottom][index][help] */
1309 {
1310     if (output_format == mon_output_monitor) {
1311         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "CLUSTER CRIT: Connection to cluster failed: %s",
1312                     pcmk_rc_str(rc));
1313         clean_up(MON_STATUS_CRIT);
1314     } else if (rc == ENOTCONN) {
1315         if (on_remote_node) {
1316             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: remote-node not connected to cluster");
1317         } else {
1318             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error: cluster is not available on this node");
1319         }
1320     } else {
1321         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Connection to cluster failed: %s", pcmk_rc_str(rc));
1322     }
1323 
1324     clean_up(pcmk_rc2exitc(rc));
1325 }
1326 
1327 static void
1328 one_shot(void)
     /* [previous][next][first][last][top][bottom][index][help] */
1329 {
1330     int rc = pcmk__status(out, cib, fence_history, show, show_opts,
1331                           options.only_node, options.only_rsc,
1332                           options.neg_location_prefix,
1333                           output_format == mon_output_monitor);
1334 
1335     if (rc == pcmk_rc_ok) {
1336         clean_up(pcmk_rc2exitc(rc));
1337     } else {
1338         clean_up_on_connection_failure(rc);
1339     }
1340 }
1341 
1342 static void
1343 exit_on_invalid_cib(void)
     /* [previous][next][first][last][top][bottom][index][help] */
1344 {
1345     if (cib != NULL) {
1346         return;
1347     }
1348 
1349     // Shouldn't really be possible
1350     g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Invalid CIB source");
1351     clean_up(CRM_EX_ERROR);
1352 }
1353 
1354 int
1355 main(int argc, char **argv)
     /* [previous][next][first][last][top][bottom][index][help] */
1356 {
1357     int rc = pcmk_rc_ok;
1358     GOptionGroup *output_group = NULL;
1359 
1360     args = pcmk__new_common_args(SUMMARY);
1361     context = build_arg_context(args, &output_group);
1362     pcmk__register_formats(output_group, formats);
1363 
1364     options.pid_file = strdup("/tmp/ClusterMon.pid");
1365     pcmk__cli_init_logging("crm_mon", 0);
1366 
1367     // Avoid needing to wait for subprocesses forked for -E/--external-agent
1368     avoid_zombies();
1369 
1370     if (pcmk__ends_with_ext(argv[0], ".cgi")) {
1371         output_format = mon_output_cgi;
1372         options.one_shot = TRUE;
1373     }
1374 
1375     processed_args = pcmk__cmdline_preproc(argv, "ehimpxEILU");
1376 
1377     fence_history_cb("--fence-history", "1", NULL, NULL);
1378 
1379     /* Set an HTML title regardless of what format we will eventually use.  This can't
1380      * be done in add_output_args.  That function is called after command line
1381      * arguments are processed in the next block, which means it'll override whatever
1382      * title the user provides.  Doing this here means the user can give their own
1383      * title on the command line.
1384      */
1385     if (!pcmk__force_args(context, &error, "%s --html-title \"Cluster Status\"",
1386                           g_get_prgname())) {
1387         return clean_up(CRM_EX_USAGE);
1388     }
1389 
1390     if (!g_option_context_parse_strv(context, &processed_args, &error)) {
1391         return clean_up(CRM_EX_USAGE);
1392     }
1393 
1394     for (int i = 0; i < args->verbosity; i++) {
1395         crm_bump_log_level(argc, argv);
1396     }
1397 
1398     if (!args->version) {
1399         if (args->quiet) {
1400             include_exclude_cb("--exclude", "times", NULL, NULL);
1401         }
1402 
1403         if (options.watch_fencing) {
1404             fence_history_cb("--fence-history", "0", NULL, NULL);
1405             options.fence_connect = TRUE;
1406         }
1407 
1408         /* create the cib-object early to be able to do further
1409          * decisions based on the cib-source
1410          */
1411         cib = cib_new();
1412 
1413         exit_on_invalid_cib();
1414 
1415         switch (cib->variant) {
1416             case cib_native:
1417                 /* cib & fencing - everything available */
1418                 use_cib_native = TRUE;
1419                 break;
1420 
1421             case cib_file:
1422                 /* Don't try to connect to fencing as we
1423                  * either don't have a running cluster or
1424                  * the fencing-information would possibly
1425                  * not match the cib data from a file.
1426                  * As we don't expect cib-updates coming
1427                  * in enforce one-shot. */
1428                 fence_history_cb("--fence-history", "0", NULL, NULL);
1429                 options.one_shot = TRUE;
1430                 break;
1431 
1432             case cib_remote:
1433                 /* updates coming in but no fencing */
1434                 fence_history_cb("--fence-history", "0", NULL, NULL);
1435                 break;
1436 
1437             case cib_undefined:
1438             case cib_database:
1439             default:
1440                 /* something is odd */
1441                 exit_on_invalid_cib();
1442                 break;
1443         }
1444 
1445         if (options.one_shot) {
1446             if (output_format == mon_output_console) {
1447                 output_format = mon_output_plain;
1448             }
1449 
1450         } else if (options.daemonize) {
1451             if (pcmk__str_eq(args->output_dest, "-", pcmk__str_null_matches|pcmk__str_casei) &&
1452                 !options.external_agent) {
1453                 g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1454                             "--daemonize requires at least one of --output-to and --external-agent");
1455                 return clean_up(CRM_EX_USAGE);
1456             }
1457 
1458         } else if (output_format == mon_output_console) {
1459 #if CURSES_ENABLED
1460             crm_enable_stderr(FALSE);
1461 #else
1462             options.one_shot = TRUE;
1463             output_format = mon_output_plain;
1464             printf("Defaulting to one-shot mode\n");
1465             printf("You need to have curses available at compile time to enable console mode\n");
1466 #endif
1467         }
1468     }
1469 
1470     reconcile_output_format(args);
1471     add_output_args();
1472 
1473     /* output_format MUST NOT BE CHANGED AFTER THIS POINT. */
1474 
1475     if (args->version && output_format == mon_output_console) {
1476         /* Use the text output format here if we are in curses mode but were given
1477          * --version.  Displaying version information uses printf, and then we
1478          *  immediately exit.  We don't want to initialize curses for that.
1479          */
1480         rc = pcmk__output_new(&out, "text", args->output_dest, argv);
1481     } else {
1482         rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
1483     }
1484 
1485     if (rc != pcmk_rc_ok) {
1486         g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_ERROR, "Error creating output format %s: %s",
1487                     args->output_ty, pcmk_rc_str(rc));
1488         return clean_up(CRM_EX_ERROR);
1489     }
1490 
1491     if (options.daemonize) {
1492         if (!options.external_agent && (output_format == mon_output_console ||
1493                                         output_format == mon_output_unset ||
1494                                         output_format == mon_output_none)) {
1495             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE,
1496                         "--daemonize requires --output-as=[html|text|xml]");
1497             return clean_up(CRM_EX_USAGE);
1498         }
1499 
1500         crm_enable_stderr(FALSE);
1501 
1502         cib_delete(cib);
1503         cib = NULL;
1504         pcmk__daemonize(crm_system_name, options.pid_file);
1505         cib = cib_new();
1506         exit_on_invalid_cib();
1507     }
1508 
1509     show = default_includes(output_format);
1510 
1511     /* Apply --include/--exclude flags we used internally.  There's no error reporting
1512      * here because this would be a programming error.
1513      */
1514     apply_include_exclude(options.includes_excludes, &error);
1515 
1516     /* And now apply any --include/--exclude flags the user gave on the command line.
1517      * These are done in a separate pass from the internal ones because we want to
1518      * make sure whatever the user specifies overrides whatever we do.
1519      */
1520     if (!apply_include_exclude(options.user_includes_excludes, &error)) {
1521         return clean_up(CRM_EX_USAGE);
1522     }
1523 
1524     /* Sync up the initial value of interactive_fence_level with whatever was set with
1525      * --include/--exclude= options.
1526      */
1527     if (pcmk_all_flags_set(show, pcmk_section_fencing_all)) {
1528         interactive_fence_level = 3;
1529     } else if (pcmk_is_set(show, pcmk_section_fence_worked)) {
1530         interactive_fence_level = 2;
1531     } else if (pcmk_any_flags_set(show, pcmk_section_fence_failed | pcmk_section_fence_pending)) {
1532         interactive_fence_level = 1;
1533     } else {
1534         interactive_fence_level = 0;
1535     }
1536 
1537     pcmk__register_lib_messages(out);
1538     crm_mon_register_messages(out);
1539     pe__register_messages(out);
1540     stonith__register_messages(out);
1541 
1542     if (args->version) {
1543         out->version(out, false);
1544         return clean_up(CRM_EX_OK);
1545     }
1546 
1547     /* Extra sanity checks when in CGI mode */
1548     if (output_format == mon_output_cgi) {
1549         if (cib->variant == cib_file) {
1550             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode used with CIB file");
1551             return clean_up(CRM_EX_USAGE);
1552         } else if (options.external_agent != NULL) {
1553             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with --external-agent");
1554             return clean_up(CRM_EX_USAGE);
1555         } else if (options.daemonize == TRUE) {
1556             g_set_error(&error, PCMK__EXITC_ERROR, CRM_EX_USAGE, "CGI mode cannot be used with -d");
1557             return clean_up(CRM_EX_USAGE);
1558         }
1559     }
1560 
1561     if (output_format == mon_output_xml || output_format == mon_output_legacy_xml) {
1562         show_opts |= pcmk_show_inactive_rscs | pcmk_show_timing;
1563 
1564         if (!options.daemonize) {
1565             options.one_shot = TRUE;
1566         }
1567     }
1568 
1569     if ((output_format == mon_output_html || output_format == mon_output_cgi) &&
1570         out->dest != stdout) {
1571         pcmk__html_add_header("meta", "http-equiv", "refresh", "content",
1572                               pcmk__itoa(options.reconnect_ms / 1000), NULL);
1573     }
1574 
1575 #ifdef PCMK__COMPAT_2_0
1576     // Keep failed action output the same as 2.0.x
1577     show_opts |= pcmk_show_failed_detail;
1578 #endif
1579 
1580     crm_info("Starting %s", crm_system_name);
1581 
1582     cib__set_output(cib, out);
1583 
1584     if (options.one_shot) {
1585         one_shot();
1586     }
1587 
1588     do {
1589         out->info(out,"Waiting until cluster is available on this node ...");
1590 
1591         rc = pacemakerd_status();
1592         if (rc == pcmk_rc_ok) {
1593             fencing_connect();
1594             rc = cib_connect();
1595         }
1596 
1597         if (rc != pcmk_rc_ok) {
1598             pcmk__sleep_ms(options.reconnect_ms);
1599 #if CURSES_ENABLED
1600             if (output_format == mon_output_console) {
1601                 clear();
1602                 refresh();
1603             }
1604 #endif
1605         } else if (output_format == mon_output_html && out->dest != stdout) {
1606             printf("Writing html to %s ...\n", args->output_dest);
1607         }
1608 
1609     } while (rc == ENOTCONN);
1610 
1611     if (rc != pcmk_rc_ok) {
1612         clean_up_on_connection_failure(rc);
1613     }
1614 
1615     set_fencing_options(interactive_fence_level);
1616     mon_refresh_display(NULL);
1617 
1618     mainloop = g_main_loop_new(NULL, FALSE);
1619 
1620     mainloop_add_signal(SIGTERM, mon_shutdown);
1621     mainloop_add_signal(SIGINT, mon_shutdown);
1622 #if CURSES_ENABLED
1623     if (output_format == mon_output_console) {
1624         ncurses_winch_handler = crm_signal_handler(SIGWINCH, mon_winresize);
1625         if (ncurses_winch_handler == SIG_DFL ||
1626             ncurses_winch_handler == SIG_IGN || ncurses_winch_handler == SIG_ERR)
1627             ncurses_winch_handler = NULL;
1628 
1629         io_channel = g_io_channel_unix_new(STDIN_FILENO);
1630         g_io_add_watch(io_channel, G_IO_IN, detect_user_input, NULL);
1631     }
1632 #endif
1633 
1634     /* When refresh_trigger->trigger is set to TRUE, call mon_refresh_display.  In
1635      * this file, that is anywhere mainloop_set_trigger is called.
1636      */
1637     refresh_trigger = mainloop_add_trigger(G_PRIORITY_LOW, mon_refresh_display, NULL);
1638 
1639     g_main_loop_run(mainloop);
1640     g_main_loop_unref(mainloop);
1641 
1642     if (io_channel != NULL) {
1643         g_io_channel_shutdown(io_channel, TRUE, NULL);
1644     }
1645 
1646     crm_info("Exiting %s", crm_system_name);
1647 
1648     return clean_up(CRM_EX_OK);
1649 }
1650 
1651 static int
1652 send_custom_trap(const char *node, const char *rsc, const char *task, int target_rc, int rc,
     /* [previous][next][first][last][top][bottom][index][help] */
1653                  int status, const char *desc)
1654 {
1655     pid_t pid;
1656 
1657     /*setenv needs chars, these are ints */
1658     char *rc_s = pcmk__itoa(rc);
1659     char *status_s = pcmk__itoa(status);
1660     char *target_rc_s = pcmk__itoa(target_rc);
1661 
1662     crm_debug("Sending external notification to '%s' via '%s'", options.external_recipient, options.external_agent);
1663 
1664     if(rsc) {
1665         setenv("CRM_notify_rsc", rsc, 1);
1666     }
1667     if (options.external_recipient) {
1668         setenv("CRM_notify_recipient", options.external_recipient, 1);
1669     }
1670     setenv("CRM_notify_node", node, 1);
1671     setenv("CRM_notify_task", task, 1);
1672     setenv("CRM_notify_desc", desc, 1);
1673     setenv("CRM_notify_rc", rc_s, 1);
1674     setenv("CRM_notify_target_rc", target_rc_s, 1);
1675     setenv("CRM_notify_status", status_s, 1);
1676 
1677     pid = fork();
1678     if (pid == -1) {
1679         crm_perror(LOG_ERR, "notification fork() failed.");
1680     }
1681     if (pid == 0) {
1682         /* crm_debug("notification: I am the child. Executing the nofitication program."); */
1683         execl(options.external_agent, options.external_agent, NULL);
1684         exit(CRM_EX_ERROR);
1685     }
1686 
1687     crm_trace("Finished running custom notification program '%s'.", options.external_agent);
1688     free(target_rc_s);
1689     free(status_s);
1690     free(rc_s);
1691     return 0;
1692 }
1693 
1694 static void
1695 handle_rsc_op(xmlNode * xml, const char *node_id)
     /* [previous][next][first][last][top][bottom][index][help] */
1696 {
1697     int rc = -1;
1698     int status = -1;
1699     int target_rc = -1;
1700     gboolean notify = TRUE;
1701 
1702     char *rsc = NULL;
1703     char *task = NULL;
1704     const char *desc = NULL;
1705     const char *magic = NULL;
1706     const char *id = NULL;
1707     const char *node = NULL;
1708 
1709     xmlNode *n = xml;
1710     xmlNode * rsc_op = xml;
1711 
1712     if(strcmp((const char*)xml->name, XML_LRM_TAG_RSC_OP) != 0) {
1713         xmlNode *cIter;
1714 
1715         for(cIter = xml->children; cIter; cIter = cIter->next) {
1716             handle_rsc_op(cIter, node_id);
1717         }
1718 
1719         return;
1720     }
1721 
1722     id = crm_element_value(rsc_op, XML_LRM_ATTR_TASK_KEY);
1723     if (id == NULL) {
1724         /* Compatibility with <= 1.1.5 */
1725         id = ID(rsc_op);
1726     }
1727 
1728     magic = crm_element_value(rsc_op, XML_ATTR_TRANSITION_MAGIC);
1729     if (magic == NULL) {
1730         /* non-change */
1731         return;
1732     }
1733 
1734     if (!decode_transition_magic(magic, NULL, NULL, NULL, &status, &rc,
1735                                  &target_rc)) {
1736         crm_err("Invalid event %s detected for %s", magic, id);
1737         return;
1738     }
1739 
1740     if (parse_op_key(id, &rsc, &task, NULL) == FALSE) {
1741         crm_err("Invalid event detected for %s", id);
1742         goto bail;
1743     }
1744 
1745     node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
1746 
1747     while (n != NULL && !pcmk__str_eq(XML_CIB_TAG_STATE, TYPE(n), pcmk__str_casei)) {
1748         n = n->parent;
1749     }
1750 
1751     if(node == NULL && n) {
1752         node = crm_element_value(n, XML_ATTR_UNAME);
1753     }
1754 
1755     if (node == NULL && n) {
1756         node = ID(n);
1757     }
1758 
1759     if (node == NULL) {
1760         node = node_id;
1761     }
1762 
1763     if (node == NULL) {
1764         crm_err("No node detected for event %s (%s)", magic, id);
1765         goto bail;
1766     }
1767 
1768     /* look up where we expected it to be? */
1769     desc = pcmk_rc_str(pcmk_rc_ok);
1770     if ((status == PCMK_EXEC_DONE) && (target_rc == rc)) {
1771         crm_notice("%s of %s on %s completed: %s", task, rsc, node, desc);
1772         if (rc == PCMK_OCF_NOT_RUNNING) {
1773             notify = FALSE;
1774         }
1775 
1776     } else if (status == PCMK_EXEC_DONE) {
1777         desc = services_ocf_exitcode_str(rc);
1778         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1779 
1780     } else {
1781         desc = pcmk_exec_status_str(status);
1782         crm_warn("%s of %s on %s failed: %s", task, rsc, node, desc);
1783     }
1784 
1785     if (notify && options.external_agent) {
1786         send_custom_trap(node, rsc, task, target_rc, rc, status, desc);
1787     }
1788   bail:
1789     free(rsc);
1790     free(task);
1791 }
1792 
1793 /* This function is just a wrapper around mainloop_set_trigger so that it can be
1794  * called from a mainloop directly.  It's simply another way of ensuring the screen
1795  * gets redrawn.
1796  */
1797 static gboolean
1798 mon_trigger_refresh(gpointer user_data)
     /* [previous][next][first][last][top][bottom][index][help] */
1799 {
1800     mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
1801     return FALSE;
1802 }
1803 
1804 static void
1805 crm_diff_update_v2(const char *event, xmlNode * msg)
     /* [previous][next][first][last][top][bottom][index][help] */
1806 {
1807     xmlNode *change = NULL;
1808     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
1809 
1810     for (change = pcmk__xml_first_child(diff); change != NULL;
1811          change = pcmk__xml_next(change)) {
1812         const char *name = NULL;
1813         const char *op = crm_element_value(change, XML_DIFF_OP);
1814         const char *xpath = crm_element_value(change, XML_DIFF_PATH);
1815         xmlNode *match = NULL;
1816         const char *node = NULL;
1817 
1818         if(op == NULL) {
1819             continue;
1820 
1821         } else if(strcmp(op, "create") == 0) {
1822             match = change->children;
1823 
1824         } else if(strcmp(op, "move") == 0) {
1825             continue;
1826 
1827         } else if(strcmp(op, "delete") == 0) {
1828             continue;
1829 
1830         } else if(strcmp(op, "modify") == 0) {
1831             match = first_named_child(change, XML_DIFF_RESULT);
1832             if(match) {
1833                 match = match->children;
1834             }
1835         }
1836 
1837         if(match) {
1838             name = (const char *)match->name;
1839         }
1840 
1841         crm_trace("Handling %s operation for %s %p, %s", op, xpath, match, name);
1842         if(xpath == NULL) {
1843             /* Version field, ignore */
1844 
1845         } else if(name == NULL) {
1846             crm_debug("No result for %s operation to %s", op, xpath);
1847             CRM_ASSERT(strcmp(op, "delete") == 0 || strcmp(op, "move") == 0);
1848 
1849         } else if(strcmp(name, XML_TAG_CIB) == 0) {
1850             xmlNode *state = NULL;
1851             xmlNode *status = first_named_child(match, XML_CIB_TAG_STATUS);
1852 
1853             for (state = pcmk__xe_first_child(status); state != NULL;
1854                  state = pcmk__xe_next(state)) {
1855 
1856                 node = crm_element_value(state, XML_ATTR_UNAME);
1857                 if (node == NULL) {
1858                     node = ID(state);
1859                 }
1860                 handle_rsc_op(state, node);
1861             }
1862 
1863         } else if(strcmp(name, XML_CIB_TAG_STATUS) == 0) {
1864             xmlNode *state = NULL;
1865 
1866             for (state = pcmk__xe_first_child(match); state != NULL;
1867                  state = pcmk__xe_next(state)) {
1868 
1869                 node = crm_element_value(state, XML_ATTR_UNAME);
1870                 if (node == NULL) {
1871                     node = ID(state);
1872                 }
1873                 handle_rsc_op(state, node);
1874             }
1875 
1876         } else if(strcmp(name, XML_CIB_TAG_STATE) == 0) {
1877             node = crm_element_value(match, XML_ATTR_UNAME);
1878             if (node == NULL) {
1879                 node = ID(match);
1880             }
1881             handle_rsc_op(match, node);
1882 
1883         } else if(strcmp(name, XML_CIB_TAG_LRM) == 0) {
1884             node = ID(match);
1885             handle_rsc_op(match, node);
1886 
1887         } else if(strcmp(name, XML_LRM_TAG_RESOURCES) == 0) {
1888             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
1889 
1890             handle_rsc_op(match, local_node);
1891             free(local_node);
1892 
1893         } else if(strcmp(name, XML_LRM_TAG_RESOURCE) == 0) {
1894             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
1895 
1896             handle_rsc_op(match, local_node);
1897             free(local_node);
1898 
1899         } else if(strcmp(name, XML_LRM_TAG_RSC_OP) == 0) {
1900             char *local_node = pcmk__xpath_node_id(xpath, "lrm");
1901 
1902             handle_rsc_op(match, local_node);
1903             free(local_node);
1904 
1905         } else {
1906             crm_trace("Ignoring %s operation for %s %p, %s", op, xpath, match, name);
1907         }
1908     }
1909 }
1910 
1911 static void
1912 crm_diff_update_v1(const char *event, xmlNode * msg)
     /* [previous][next][first][last][top][bottom][index][help] */
1913 {
1914     /* Process operation updates */
1915     xmlXPathObject *xpathObj = xpath_search(msg,
1916                                             "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED
1917                                             "//" XML_LRM_TAG_RSC_OP);
1918     int lpc = 0, max = numXpathResults(xpathObj);
1919 
1920     for (lpc = 0; lpc < max; lpc++) {
1921         xmlNode *rsc_op = getXpathResult(xpathObj, lpc);
1922 
1923         handle_rsc_op(rsc_op, NULL);
1924     }
1925     freeXpathObject(xpathObj);
1926 }
1927 
1928 static void
1929 crm_diff_update(const char *event, xmlNode * msg)
     /* [previous][next][first][last][top][bottom][index][help] */
1930 {
1931     int rc = -1;
1932     static bool stale = FALSE;
1933     gboolean cib_updated = FALSE;
1934     xmlNode *diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
1935 
1936     out->progress(out, false);
1937 
1938     if (current_cib != NULL) {
1939         rc = xml_apply_patchset(current_cib, diff, TRUE);
1940 
1941         switch (rc) {
1942             case -pcmk_err_diff_resync:
1943             case -pcmk_err_diff_failed:
1944                 crm_notice("[%s] Patch aborted: %s (%d)", event, pcmk_strerror(rc), rc);
1945                 free_xml(current_cib); current_cib = NULL;
1946                 break;
1947             case pcmk_ok:
1948                 cib_updated = TRUE;
1949                 break;
1950             default:
1951                 crm_notice("[%s] ABORTED: %s (%d)", event, pcmk_strerror(rc), rc);
1952                 free_xml(current_cib); current_cib = NULL;
1953         }
1954     }
1955 
1956     if (current_cib == NULL) {
1957         crm_trace("Re-requesting the full cib");
1958         cib->cmds->query(cib, NULL, &current_cib, cib_scope_local | cib_sync_call);
1959     }
1960 
1961     if (options.external_agent) {
1962         int format = 0;
1963         crm_element_value_int(diff, "format", &format);
1964         switch(format) {
1965             case 1:
1966                 crm_diff_update_v1(event, msg);
1967                 break;
1968             case 2:
1969                 crm_diff_update_v2(event, msg);
1970                 break;
1971             default:
1972                 crm_err("Unknown patch format: %d", format);
1973         }
1974     }
1975 
1976     if (current_cib == NULL) {
1977         if(!stale) {
1978             out->info(out, "--- Stale data ---");
1979         }
1980         stale = TRUE;
1981         return;
1982     }
1983 
1984     stale = FALSE;
1985     refresh_after_event(cib_updated, FALSE);
1986 }
1987 
1988 static int
1989 mon_refresh_display(gpointer user_data)
     /* [previous][next][first][last][top][bottom][index][help] */
1990 {
1991     int rc = pcmk_rc_ok;
1992 
1993     last_refresh = time(NULL);
1994 
1995     if (output_format == mon_output_none || output_format == mon_output_unset) {
1996         return G_SOURCE_REMOVE;
1997     }
1998 
1999     if (fence_history == pcmk__fence_history_full &&
2000         !pcmk_all_flags_set(show, pcmk_section_fencing_all) &&
2001         output_format != mon_output_xml) {
2002         fence_history = pcmk__fence_history_reduced;
2003      }
2004 
2005     if (out->dest != stdout) {
2006         out->reset(out);
2007     }
2008 
2009     rc = pcmk__output_cluster_status(out, st, cib, current_cib, fence_history,
2010                                      show, show_opts, options.only_node,
2011                                      options.only_rsc, options.neg_location_prefix,
2012                                      output_format == mon_output_monitor);
2013 
2014     if (output_format == mon_output_monitor && rc != pcmk_rc_ok) {
2015         clean_up(MON_STATUS_WARN);
2016         return G_SOURCE_REMOVE;
2017     } else if (rc == pcmk_rc_schema_validation) {
2018         clean_up(CRM_EX_CONFIG);
2019         return G_SOURCE_REMOVE;
2020     }
2021 
2022     if (out->dest != stdout) {
2023         out->finish(out, CRM_EX_OK, true, NULL);
2024     }
2025 
2026     return G_SOURCE_CONTINUE;
2027 }
2028 
2029 /* This function is called for fencing events (see fencing_connect for which ones) when
2030  * --watch-fencing is used on the command line.
2031  */
2032 static void
2033 mon_st_callback_event(stonith_t * st, stonith_event_t * e)
     /* [previous][next][first][last][top][bottom][index][help] */
2034 {
2035     if (st->state == stonith_disconnected) {
2036         /* disconnect cib as well and have everything reconnect */
2037         mon_cib_connection_destroy(NULL);
2038     } else if (options.external_agent) {
2039         char *desc = stonith__event_description(e);
2040 
2041         send_custom_trap(e->target, NULL, e->operation, pcmk_ok, e->result, 0, desc);
2042         free(desc);
2043     }
2044 }
2045 
2046 /* Cause the screen to be redrawn (via mainloop_set_trigger) when various conditions are met:
2047  *
2048  * - If the last update occurred more than reconnect_ms ago (defaults to 5s, but
2049  *   can be changed via the -i command line option), or
2050  * - After every 10 CIB updates, or
2051  * - If it's been 2s since the last update
2052  *
2053  * This function sounds like it would be more broadly useful, but it is only called when a
2054  * fencing event is received or a CIB diff occurrs.
2055  */
2056 static void
2057 refresh_after_event(gboolean data_updated, gboolean enforce)
     /* [previous][next][first][last][top][bottom][index][help] */
2058 {
2059     static int updates = 0;
2060     time_t now = time(NULL);
2061 
2062     if (data_updated) {
2063         updates++;
2064     }
2065 
2066     if(refresh_timer == NULL) {
2067         refresh_timer = mainloop_timer_add("refresh", 2000, FALSE, mon_trigger_refresh, NULL);
2068     }
2069 
2070     if (reconnect_timer > 0) {
2071         /* we will receive a refresh request after successful reconnect */
2072         mainloop_timer_stop(refresh_timer);
2073         return;
2074     }
2075 
2076     /* as we're not handling initial failure of fencer-connection as
2077      * fatal give it a retry here
2078      * not getting here if cib-reconnection is already on the way
2079      */
2080     fencing_connect();
2081 
2082     if (enforce ||
2083         ((now - last_refresh) > (options.reconnect_ms / 1000)) ||
2084         updates >= 10) {
2085         mainloop_set_trigger((crm_trigger_t *) refresh_trigger);
2086         mainloop_timer_stop(refresh_timer);
2087         updates = 0;
2088 
2089     } else {
2090         mainloop_timer_start(refresh_timer);
2091     }
2092 }
2093 
2094 /* This function is called for fencing events (see fencing_connect for which ones) when
2095  * --watch-fencing is NOT used on the command line.
2096  */
2097 static void
2098 mon_st_callback_display(stonith_t * st, stonith_event_t * e)
     /* [previous][next][first][last][top][bottom][index][help] */
2099 {
2100     if (st->state == stonith_disconnected) {
2101         /* disconnect cib as well and have everything reconnect */
2102         mon_cib_connection_destroy(NULL);
2103     } else {
2104         out->progress(out, false);
2105         refresh_after_event(TRUE, FALSE);
2106     }
2107 }
2108 
2109 static void
2110 clean_up_fencing_connection(void)
     /* [previous][next][first][last][top][bottom][index][help] */
2111 {
2112     if (st == NULL) {
2113         return;
2114     }
2115 
2116     if (st->state != stonith_disconnected) {
2117         st->cmds->remove_notification(st, NULL);
2118         st->cmds->disconnect(st);
2119     }
2120 
2121     stonith_api_delete(st);
2122     st = NULL;
2123 }
2124 
2125 /*
2126  * De-init ncurses, disconnect from the CIB manager, disconnect fencing,
2127  * deallocate memory and show usage-message if requested.
2128  *
2129  * We don't actually return, but nominally returning crm_exit_t allows a usage
2130  * like "return clean_up(exit_code);" which helps static analysis understand the
2131  * code flow.
2132  */
2133 static crm_exit_t
2134 clean_up(crm_exit_t exit_code)
     /* [previous][next][first][last][top][bottom][index][help] */
2135 {
2136     /* Quitting crm_mon is much more complicated than it ought to be. */
2137 
2138     /* (1) Close connections, free things, etc. */
2139     cib__clean_up_connection(&cib);
2140     clean_up_fencing_connection();
2141     free(options.neg_location_prefix);
2142     free(options.only_node);
2143     free(options.only_rsc);
2144     free(options.pid_file);
2145     g_slist_free_full(options.includes_excludes, free);
2146 
2147     g_strfreev(processed_args);
2148 
2149     /* (2) If this is abnormal termination and we're in curses mode, shut down
2150      * curses first.  Any messages displayed to the screen before curses is shut
2151      * down will be lost because doing the shut down will also restore the
2152      * screen to whatever it looked like before crm_mon was started.
2153      */
2154     if ((error != NULL || exit_code == CRM_EX_USAGE) && output_format == mon_output_console) {
2155         out->finish(out, exit_code, false, NULL);
2156         pcmk__output_free(out);
2157         out = NULL;
2158     }
2159 
2160     /* (3) If this is a command line usage related failure, print the usage
2161      * message.
2162      */
2163     if (exit_code == CRM_EX_USAGE && (output_format == mon_output_console || output_format == mon_output_plain)) {
2164         char *help = g_option_context_get_help(context, TRUE, NULL);
2165 
2166         fprintf(stderr, "%s", help);
2167         g_free(help);
2168     }
2169 
2170     pcmk__free_arg_context(context);
2171 
2172     /* (4) If this is any kind of error, print the error out and exit.  Make
2173      * sure to handle situations both before and after formatted output is
2174      * set up.  We want errors to appear formatted if at all possible.
2175      */
2176     if (error != NULL) {
2177         if (out != NULL) {
2178             out->err(out, "%s: %s", g_get_prgname(), error->message);
2179             out->finish(out, exit_code, true, NULL);
2180             pcmk__output_free(out);
2181         } else {
2182             fprintf(stderr, "%s: %s\n", g_get_prgname(), error->message);
2183         }
2184 
2185         g_clear_error(&error);
2186         crm_exit(exit_code);
2187     }
2188 
2189     /* (5) Print formatted output to the screen if we made it far enough in
2190      * crm_mon to be able to do so.
2191      */
2192     if (out != NULL) {
2193         if (!options.daemonize) {
2194             out->finish(out, exit_code, true, NULL);
2195         }
2196 
2197         pcmk__output_free(out);
2198         pcmk__unregister_formats();
2199     }
2200 
2201     crm_exit(exit_code);
2202 }

/* [previous][next][first][last][top][bottom][index][help] */